From cbcd1ebd328afeb7eeca281a608a64e5a8b69261 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Mon, 14 Sep 2020 10:25:59 -0400 Subject: [PATCH 1/5] [Mappings editor] Add support for wildcard field type (#76574) --- .../ignore_above_parameter.tsx | 39 +++++++++++++++++++ .../document_fields/field_parameters/index.ts | 2 + .../fields/field_types/flattened_type.tsx | 26 ++----------- .../fields/field_types/index.ts | 2 + .../fields/field_types/keyword_type.tsx | 21 ++-------- .../fields/field_types/wildcard_type.tsx | 29 ++++++++++++++ .../constants/data_types_definition.tsx | 18 +++++++++ .../mappings_editor/types/document_fields.ts | 1 + .../translations/translations/ja-JP.json | 5 --- .../translations/translations/zh-CN.json | 5 --- 10 files changed, 98 insertions(+), 50 deletions(-) create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/ignore_above_parameter.tsx create mode 100644 x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/wildcard_type.tsx diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/ignore_above_parameter.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/ignore_above_parameter.tsx new file mode 100644 index 0000000000000..48a8e42f5065d --- /dev/null +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/ignore_above_parameter.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; + +import { i18n } from '@kbn/i18n'; + +import { documentationService } from '../../../../../services/documentation'; +import { getFieldConfig } from '../../../lib'; +import { UseField, Field } from '../../../shared_imports'; +import { EditFieldFormRow } from '../fields/edit_field'; + +interface Props { + defaultToggleValue: boolean; +} + +export const IgnoreAboveParameter: FunctionComponent = ({ defaultToggleValue }) => ( + + + +); diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts index 805a6b6ece705..a2d5c7c8d5308 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/field_parameters/index.ts @@ -69,6 +69,8 @@ export * from './other_type_name_parameter'; export * from './other_type_json_parameter'; +export * from './ignore_above_parameter'; + export const PARAMETER_SERIALIZERS = [relationsSerializer, dynamicSerializer]; export const PARAMETER_DESERIALIZERS = [relationsDeserializer, dynamicDeserializer]; diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/flattened_type.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/flattened_type.tsx index 7c8ac86f14153..e96426ece27e8 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/flattened_type.tsx +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/flattened_type.tsx @@ -6,7 +6,6 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; -import { documentationService } from '../../../../../../services/documentation'; import { NormalizedField, Field as FieldType } from '../../../../types'; import { UseField, Field } from '../../../../shared_imports'; import { getFieldConfig } from '../../../../lib'; @@ -19,6 +18,7 @@ import { NullValueParameter, SimilarityParameter, SplitQueriesOnWhitespaceParameter, + IgnoreAboveParameter, } from '../../field_parameters'; import { BasicParametersSection, EditFieldFormRow, AdvancedParametersSection } from '../edit_field'; @@ -29,6 +29,7 @@ interface Props { const getDefaultToggleValue = (param: string, field: FieldType) => { switch (param) { case 'boost': + case 'ignore_above': case 'similarity': { return field[param] !== undefined && field[param] !== getFieldConfig(param).defaultValue; } @@ -66,28 +67,9 @@ export const FlattenedType = React.memo(({ field }: Props) => { - {/* ignore_above */} - - - + /> diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts index 62aa2d1eb0b3b..d84d9c6ea40cf 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/index.ts @@ -29,6 +29,7 @@ import { OtherType } from './other_type'; import { NestedType } from './nested_type'; import { JoinType } from './join_type'; import { RankFeatureType } from './rank_feature_type'; +import { WildcardType } from './wildcard_type'; const typeToParametersFormMap: { [key in DataType]?: ComponentType } = { alias: AliasType, @@ -54,6 +55,7 @@ const typeToParametersFormMap: { [key in DataType]?: ComponentType } = { nested: NestedType, join: JoinType, rank_feature: RankFeatureType, + wildcard: WildcardType, }; export const getParametersFormForType = ( diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/keyword_type.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/keyword_type.tsx index 43377357f1e6f..dc4f4b3ba5ff1 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/keyword_type.tsx +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/keyword_type.tsx @@ -23,6 +23,7 @@ import { SimilarityParameter, CopyToParameter, SplitQueriesOnWhitespaceParameter, + IgnoreAboveParameter, } from '../../field_parameters'; import { BasicParametersSection, EditFieldFormRow, AdvancedParametersSection } from '../edit_field'; @@ -79,25 +80,9 @@ export const KeywordType = ({ field }: Props) => { - {/* ignore_above */} - - - + /> diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/wildcard_type.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/wildcard_type.tsx new file mode 100644 index 0000000000000..825b9e17c8d2c --- /dev/null +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/field_types/wildcard_type.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; + +import { NormalizedField, Field as FieldType, ParameterName } from '../../../../types'; +import { getFieldConfig } from '../../../../lib'; +import { IgnoreAboveParameter } from '../../field_parameters'; +import { AdvancedParametersSection } from '../edit_field'; + +interface Props { + field: NormalizedField; +} + +const getDefaultToggleValue = (param: ParameterName, field: FieldType) => { + return field[param] !== undefined && field[param] !== getFieldConfig(param).defaultValue; +}; + +export const WildcardType = ({ field }: Props) => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx index edfb6903a8585..a8844c7a9b270 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx @@ -784,6 +784,23 @@ export const TYPE_DEFINITION: { [key in DataType]: DataTypeDefinition } = {

), }, + wildcard: { + label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.wildcardDescription', { + defaultMessage: 'Wildcard', + }), + value: 'wildcard', + documentation: { + main: '/keyword.html#wildcard-field-type', + }, + description: () => ( +

+ +

+ ), + }, other: { label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.otherDescription', { defaultMessage: 'Other', @@ -825,6 +842,7 @@ export const MAIN_TYPES: MainType[] = [ 'shape', 'text', 'token_count', + 'wildcard', 'other', ]; diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts index 131ce08a87ad7..fd0e4ed32bfe8 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts @@ -59,6 +59,7 @@ export type MainType = | 'geo_point' | 'geo_shape' | 'token_count' + | 'wildcard' /** * 'other' is a special type that only exists inside of MappingsEditor as a placeholder * for undocumented field types. diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 70e7ed3f5b784..3938ed163f6cc 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -7574,7 +7574,6 @@ "xpack.idxMgmt.mappingsEditor.fielddata.frequencyFilterPercentageFieldLabel": "パーセンテージベースの頻度範囲", "xpack.idxMgmt.mappingsEditor.fielddata.useAbsoluteValuesFieldLabel": "絶対値の使用", "xpack.idxMgmt.mappingsEditor.fieldsTabLabel": "マッピングされたフィールド", - "xpack.idxMgmt.mappingsEditor.flattened.ignoreAboveDocLinkText": "上記ドキュメントの無視", "xpack.idxMgmt.mappingsEditor.formatDocLinkText": "フォーマットのドキュメンテーション", "xpack.idxMgmt.mappingsEditor.formatFieldLabel": "フォーマット", "xpack.idxMgmt.mappingsEditor.formatHelpText": "{dateSyntax}構文を使用し、カスタムフォーマットを指定します。", @@ -7705,10 +7704,6 @@ "xpack.idxMgmt.mappingsEditor.joinType.relationshipTable.parentFieldAriaLabel": "親フィールド", "xpack.idxMgmt.mappingsEditor.joinType.relationshipTable.removeRelationshipTooltipLabel": "関係を削除", "xpack.idxMgmt.mappingsEditor.largestShingleSizeFieldLabel": "最大シングルサイズ", - "xpack.idxMgmt.mappingsEditor.leafLengthLimitFieldDescription": "特定の長さ以上のリーフ値のインデックスを無効化。これは、Luceneの文字制限(8,191 UTF-8 文字)に対する保護に役立ちます。", - "xpack.idxMgmt.mappingsEditor.leafLengthLimitFieldTitle": "長さ制限の設定", - "xpack.idxMgmt.mappingsEditor.lengthLimitFieldDescription": "この値よりも長い文字列はインデックスされません。これは、Luceneの文字制限(8,191 UTF-8 文字)に対する保護に役立ちます。", - "xpack.idxMgmt.mappingsEditor.lengthLimitFieldTitle": "長さ制限の設定", "xpack.idxMgmt.mappingsEditor.loadFromJsonButtonLabel": "JSONの読み込み", "xpack.idxMgmt.mappingsEditor.loadJsonModal.acceptWarningLabel": "読み込みの続行", "xpack.idxMgmt.mappingsEditor.loadJsonModal.cancelButtonLabel": "キャンセル", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 861579e439d8d..c8eefb45ea9f5 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -7577,7 +7577,6 @@ "xpack.idxMgmt.mappingsEditor.fielddata.frequencyFilterPercentageFieldLabel": "基于百分比的频率范围", "xpack.idxMgmt.mappingsEditor.fielddata.useAbsoluteValuesFieldLabel": "使用绝对值", "xpack.idxMgmt.mappingsEditor.fieldsTabLabel": "已映射字段", - "xpack.idxMgmt.mappingsEditor.flattened.ignoreAboveDocLinkText": "“忽略上述”文档", "xpack.idxMgmt.mappingsEditor.formatDocLinkText": "“格式”文档", "xpack.idxMgmt.mappingsEditor.formatFieldLabel": "格式", "xpack.idxMgmt.mappingsEditor.formatHelpText": "使用 {dateSyntax} 语法指定定制格式。", @@ -7708,10 +7707,6 @@ "xpack.idxMgmt.mappingsEditor.joinType.relationshipTable.parentFieldAriaLabel": "父项字段", "xpack.idxMgmt.mappingsEditor.joinType.relationshipTable.removeRelationshipTooltipLabel": "移除关系", "xpack.idxMgmt.mappingsEditor.largestShingleSizeFieldLabel": "最大瓦形大小", - "xpack.idxMgmt.mappingsEditor.leafLengthLimitFieldDescription": "如果叶值超过一定长度，则阻止叶值索引。这用于防止超出 Lucene 的字词字符长度限制，即 8,191 个 UTF-8 字符。", - "xpack.idxMgmt.mappingsEditor.leafLengthLimitFieldTitle": "设置长度限制", - "xpack.idxMgmt.mappingsEditor.lengthLimitFieldDescription": "将不索引超过此值的字符串。这用于防止超出 Lucene 的字词字符长度限制，即 8,191 个 UTF-8 字符。", - "xpack.idxMgmt.mappingsEditor.lengthLimitFieldTitle": "设置长度限制", "xpack.idxMgmt.mappingsEditor.loadFromJsonButtonLabel": "加载 JSON", "xpack.idxMgmt.mappingsEditor.loadJsonModal.acceptWarningLabel": "继续加载", "xpack.idxMgmt.mappingsEditor.loadJsonModal.cancelButtonLabel": "取消", From dd1822047c86239769f17d89799ace47c58e6ee6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 14 Sep 2020 16:31:23 +0200 Subject: [PATCH 2/5] [ML] Transforms: API schemas and integration tests (#75164) - Adds schema definitions to transform API endpoints and adds API integration tests. - The type definitions based on the schema definitions can be used on the client side too. - Adds apidoc documentation. --- x-pack/plugins/ml/common/index.ts | 7 + x-pack/plugins/ml/common/types/es_client.ts | 25 ++ .../application/components/data_grid/index.ts | 1 - .../application/components/data_grid/types.ts | 11 - .../common/get_index_data.ts | 3 +- .../hooks/use_index_data.ts | 2 +- .../common/api_schemas/audit_messages.ts | 9 + .../transform/common/api_schemas/common.ts | 48 +++ .../common/api_schemas/delete_transforms.ts | 37 ++ .../common/api_schemas/field_histograms.ts | 19 + .../common/api_schemas/start_transforms.ts | 13 + .../common/api_schemas/stop_transforms.ts | 19 + .../common/api_schemas/transforms.ts | 127 ++++++ .../common/api_schemas/transforms_stats.ts | 21 + .../common/api_schemas/type_guards.ts | 114 +++++ .../common/api_schemas/update_transforms.ts | 24 ++ x-pack/plugins/transform/common/constants.ts | 21 + x-pack/plugins/transform/common/index.ts | 58 --- .../transform/common/shared_imports.ts | 7 + .../transform/common/types/aggregations.ts | 7 + .../types/es_index.ts} | 0 .../plugins/transform/common/types/fields.ts | 7 + .../transform/common/types/pivot_aggs.ts | 31 ++ .../transform/common/types/pivot_group_by.ts | 33 ++ .../transform/common/types/privileges.ts | 14 + .../transform/common/types/transform.ts | 17 + .../transform/common/types/transform_stats.ts | 62 +++ .../public/__mocks__/shared_imports.ts | 1 - .../public/app/common/aggregations.ts | 2 +- .../public/app/common/data_grid.test.ts | 12 +- .../transform/public/app/common/data_grid.ts | 5 +- .../transform/public/app/common/fields.ts | 2 +- .../transform/public/app/common/index.ts | 30 +- .../transform/public/app/common/pivot_aggs.ts | 33 +- .../public/app/common/pivot_group_by.ts | 31 +- .../public/app/common/pivot_preview.ts | 29 -- .../public/app/common/request.test.ts | 20 +- .../transform/public/app/common/request.ts | 99 +++-- .../transform/public/app/common/transform.ts | 43 +- .../public/app/common/transform_list.ts | 5 +- .../public/app/common/transform_stats.ts | 56 +-- .../public/app/hooks/__mocks__/use_api.ts | 185 +++++++-- .../transform/public/app/hooks/use_api.ts | 222 +++++++--- .../public/app/hooks/use_delete_transform.tsx | 284 ++++++------- .../public/app/hooks/use_get_transforms.ts | 122 +++--- .../public/app/hooks/use_index_data.ts | 56 +-- .../public/app/hooks/use_pivot_data.ts | 68 +-- ...t_transform.ts => use_start_transform.tsx} | 38 +- ...op_transform.ts => use_stop_transform.tsx} | 38 +- .../components/authorization_provider.tsx | 2 +- .../lib/authorization/components/common.ts | 2 +- .../components/with_privileges.tsx | 2 +- .../clone_transform_section.tsx | 44 +- .../aggregation_list/agg_label_form.test.tsx | 5 +- .../aggregation_list/agg_label_form.tsx | 3 +- .../aggregation_list/list_form.test.tsx | 4 +- .../components/aggregation_list/list_form.tsx | 3 +- .../aggregation_list/list_summary.test.tsx | 4 +- .../aggregation_list/list_summary.tsx | 4 +- .../aggregation_list/popover_form.test.tsx | 5 +- .../aggregation_list/popover_form.tsx | 9 +- .../group_by_list/group_by_label_form.tsx | 3 +- .../components/group_by_list/list_form.tsx | 3 +- .../group_by_list/popover_form.test.tsx | 4 +- .../components/group_by_list/popover_form.tsx | 2 +- .../step_create/step_create_form.tsx | 135 +++--- .../apply_transform_config_to_define_state.ts | 8 +- .../components/filter_term_form.tsx | 21 +- .../step_define/common/get_agg_form_config.ts | 8 +- .../get_agg_name_conflict_toast_messages.ts | 5 +- .../common/get_default_aggregation_config.ts | 8 +- .../common/get_default_group_by_config.ts | 8 +- .../components/step_define/common/types.ts | 4 +- .../hooks/use_advanced_pivot_editor.ts | 4 +- .../hooks/use_advanced_source_editor.ts | 4 +- .../step_define/hooks/use_pivot_config.ts | 2 +- .../step_define/hooks/use_step_define_form.ts | 6 +- .../step_define/step_define_form.test.tsx | 3 +- .../step_define/step_define_form.tsx | 9 +- .../step_define/step_define_summary.test.tsx | 3 +- .../step_define/step_define_summary.tsx | 4 +- .../step_details/step_details_form.tsx | 78 ++-- .../components/wizard/wizard.tsx | 6 +- .../action_delete/delete_action_name.tsx | 6 +- .../action_delete/use_delete_action.tsx | 13 +- .../action_edit/use_edit_action.tsx | 4 +- .../action_start/start_action_name.tsx | 2 +- .../action_start/use_start_action.tsx | 4 +- .../action_stop/stop_action_name.tsx | 2 +- .../action_stop/use_stop_action.tsx | 9 +- .../edit_transform_flyout.tsx | 35 +- .../use_edit_transform_flyout.test.ts | 6 +- .../use_edit_transform_flyout.ts | 53 +-- .../components/transform_list/common.test.ts | 2 +- .../expanded_row_messages_pane.tsx | 42 +- .../expanded_row_preview_pane.tsx | 3 +- .../transform_list/transform_list.tsx | 8 +- .../transform_list/transform_search_bar.tsx | 4 +- .../transform_list/transforms_stats_bar.tsx | 4 +- .../components/transform_list/use_columns.tsx | 27 +- .../transform/public/shared_imports.ts | 1 - x-pack/plugins/transform/server/README.md | 19 + .../server/routes/api/error_utils.ts | 13 +- .../server/routes/api/field_histograms.ts | 45 +- .../transform/server/routes/api/privileges.ts | 2 +- .../transform/server/routes/api/schema.ts | 49 --- .../transform/server/routes/api/transforms.ts | 391 ++++++++++++------ .../routes/api/transforms_audit_messages.ts | 26 +- .../transform/server/routes/apidoc.json | 21 + .../transform/server/services/license.ts | 4 +- x-pack/plugins/transform/tsconfig.json | 3 + x-pack/run_functional_tests.sh | 3 - .../api_integration/apis/transform/common.ts | 30 ++ .../apis/transform/delete_transforms.ts | 133 +++--- .../api_integration/apis/transform/index.ts | 6 + .../apis/transform/start_transforms.ts | 164 ++++++++ .../apis/transform/stop_transforms.ts | 197 +++++++++ .../apis/transform/transforms.ts | 165 ++++++++ .../apis/transform/transforms_preview.ts | 72 ++++ .../apis/transform/transforms_stats.ts | 101 +++++ .../apis/transform/transforms_update.ts | 150 +++++++ .../test/functional/apps/transform/cloning.ts | 6 +- .../apps/transform/creation_index_pattern.ts | 6 +- .../apps/transform/creation_saved_search.ts | 4 +- .../test/functional/apps/transform/editing.ts | 10 +- .../test/functional/services/transform/api.ts | 80 +++- .../services/transform/transform_table.ts | 2 +- 127 files changed, 3098 insertions(+), 1362 deletions(-) create mode 100644 x-pack/plugins/ml/common/index.ts create mode 100644 x-pack/plugins/ml/common/types/es_client.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/audit_messages.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/common.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/delete_transforms.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/field_histograms.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/start_transforms.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/stop_transforms.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/transforms.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/transforms_stats.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/type_guards.ts create mode 100644 x-pack/plugins/transform/common/api_schemas/update_transforms.ts delete mode 100644 x-pack/plugins/transform/common/index.ts create mode 100644 x-pack/plugins/transform/common/shared_imports.ts create mode 100644 x-pack/plugins/transform/common/types/aggregations.ts rename x-pack/plugins/transform/{public/app/hooks/use_api_types.ts => common/types/es_index.ts} (100%) create mode 100644 x-pack/plugins/transform/common/types/fields.ts create mode 100644 x-pack/plugins/transform/common/types/pivot_aggs.ts create mode 100644 x-pack/plugins/transform/common/types/pivot_group_by.ts create mode 100644 x-pack/plugins/transform/common/types/privileges.ts create mode 100644 x-pack/plugins/transform/common/types/transform.ts create mode 100644 x-pack/plugins/transform/common/types/transform_stats.ts delete mode 100644 x-pack/plugins/transform/public/app/common/pivot_preview.ts rename x-pack/plugins/transform/public/app/hooks/{use_start_transform.ts => use_start_transform.tsx} (52%) rename x-pack/plugins/transform/public/app/hooks/{use_stop_transform.ts => use_stop_transform.tsx} (53%) create mode 100644 x-pack/plugins/transform/server/README.md delete mode 100644 x-pack/plugins/transform/server/routes/api/schema.ts create mode 100644 x-pack/plugins/transform/server/routes/apidoc.json create mode 100644 x-pack/plugins/transform/tsconfig.json delete mode 100755 x-pack/run_functional_tests.sh create mode 100644 x-pack/test/api_integration/apis/transform/common.ts create mode 100644 x-pack/test/api_integration/apis/transform/start_transforms.ts create mode 100644 x-pack/test/api_integration/apis/transform/stop_transforms.ts create mode 100644 x-pack/test/api_integration/apis/transform/transforms.ts create mode 100644 x-pack/test/api_integration/apis/transform/transforms_preview.ts create mode 100644 x-pack/test/api_integration/apis/transform/transforms_stats.ts create mode 100644 x-pack/test/api_integration/apis/transform/transforms_update.ts diff --git a/x-pack/plugins/ml/common/index.ts b/x-pack/plugins/ml/common/index.ts new file mode 100644 index 0000000000000..791a7de48f36f --- /dev/null +++ b/x-pack/plugins/ml/common/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { SearchResponse7 } from './types/es_client'; diff --git a/x-pack/plugins/ml/common/types/es_client.ts b/x-pack/plugins/ml/common/types/es_client.ts new file mode 100644 index 0000000000000..d9ca9a3b584ab --- /dev/null +++ b/x-pack/plugins/ml/common/types/es_client.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse, ShardsResponse } from 'elasticsearch'; + +// The types specified in `@types/elasticsearch` are out of date and still have `total: number`. +interface SearchResponse7Hits { + hits: SearchResponse['hits']['hits']; + max_score: number; + total: { + value: number; + relation: string; + }; +} +export interface SearchResponse7 { + took: number; + timed_out: boolean; + _scroll_id?: string; + _shards: ShardsResponse; + hits: SearchResponse7Hits; + aggregations?: any; +} diff --git a/x-pack/plugins/ml/public/application/components/data_grid/index.ts b/x-pack/plugins/ml/public/application/components/data_grid/index.ts index 4bbd3595e5a7e..633d70687dd27 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts @@ -19,7 +19,6 @@ export { DataGridItem, EsSorting, RenderCellValue, - SearchResponse7, UseDataGridReturnType, UseIndexDataReturnType, } from './types'; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/types.ts b/x-pack/plugins/ml/public/application/components/data_grid/types.ts index f9ee8c37fabf7..22fff0f6e0b93 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/types.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/types.ts @@ -5,7 +5,6 @@ */ import { Dispatch, SetStateAction } from 'react'; -import { SearchResponse } from 'elasticsearch'; import { EuiDataGridPaginationProps, EuiDataGridSorting, EuiDataGridColumn } from '@elastic/eui'; @@ -43,16 +42,6 @@ export type EsSorting = Dictionary<{ order: 'asc' | 'desc'; }>; -// The types specified in `@types/elasticsearch` are out of date and still have `total: number`. -export interface SearchResponse7 extends SearchResponse { - hits: SearchResponse['hits'] & { - total: { - value: number; - relation: string; - }; - }; -} - export interface UseIndexDataReturnType extends Pick< UseDataGridReturnType, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts index 53c0f02fd9a80..361a79d42214d 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts @@ -4,9 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ +import type { SearchResponse7 } from '../../../../common/types/es_client'; import { extractErrorMessage } from '../../../../common/util/errors'; -import { EsSorting, SearchResponse7, UseDataGridReturnType } from '../../components/data_grid'; +import { EsSorting, UseDataGridReturnType } from '../../components/data_grid'; import { ml } from '../../services/ml_api_service'; import { isKeywordAndTextType } from '../common/fields'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index ea958c8c4a3a3..74d45b86c8c4d 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -22,9 +22,9 @@ import { useDataGrid, useRenderCellValue, EsSorting, - SearchResponse7, UseIndexDataReturnType, } from '../../../../components/data_grid'; +import type { SearchResponse7 } from '../../../../../../common/types/es_client'; import { extractErrorMessage } from '../../../../../../common/util/errors'; import { INDEX_STATUS } from '../../../common/analytics'; import { ml } from '../../../../services/ml_api_service'; diff --git a/x-pack/plugins/transform/common/api_schemas/audit_messages.ts b/x-pack/plugins/transform/common/api_schemas/audit_messages.ts new file mode 100644 index 0000000000000..76e63af262674 --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/audit_messages.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TransformMessage } from '../types/messages'; + +export type GetTransformsAuditMessagesResponseSchema = TransformMessage[]; diff --git a/x-pack/plugins/transform/common/api_schemas/common.ts b/x-pack/plugins/transform/common/api_schemas/common.ts new file mode 100644 index 0000000000000..80b14ce6adee8 --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/common.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +import { TRANSFORM_STATE } from '../constants'; + +export const transformIdsSchema = schema.arrayOf( + schema.object({ + id: schema.string(), + }) +); + +export type TransformIdsSchema = TypeOf; + +export const transformStateSchema = schema.oneOf([ + schema.literal(TRANSFORM_STATE.ABORTING), + schema.literal(TRANSFORM_STATE.FAILED), + schema.literal(TRANSFORM_STATE.INDEXING), + schema.literal(TRANSFORM_STATE.STARTED), + schema.literal(TRANSFORM_STATE.STOPPED), + schema.literal(TRANSFORM_STATE.STOPPING), +]); + +export const indexPatternTitleSchema = schema.object({ + /** Title of the index pattern for which to return stats. */ + indexPatternTitle: schema.string(), +}); + +export type IndexPatternTitleSchema = TypeOf; + +export const transformIdParamSchema = schema.object({ + transformId: schema.string(), +}); + +export type TransformIdParamSchema = TypeOf; + +export interface ResponseStatus { + success: boolean; + error?: any; +} + +export interface CommonResponseStatusSchema { + [key: string]: ResponseStatus; +} diff --git a/x-pack/plugins/transform/common/api_schemas/delete_transforms.ts b/x-pack/plugins/transform/common/api_schemas/delete_transforms.ts new file mode 100644 index 0000000000000..c4d1a1f5f7587 --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/delete_transforms.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +import { transformStateSchema, ResponseStatus } from './common'; + +export const deleteTransformsRequestSchema = schema.object({ + /** + * Delete Transform & Destination Index + */ + transformsInfo: schema.arrayOf( + schema.object({ + id: schema.string(), + state: transformStateSchema, + }) + ), + deleteDestIndex: schema.maybe(schema.boolean()), + deleteDestIndexPattern: schema.maybe(schema.boolean()), + forceDelete: schema.maybe(schema.boolean()), +}); + +export type DeleteTransformsRequestSchema = TypeOf; + +export interface DeleteTransformStatus { + transformDeleted: ResponseStatus; + destIndexDeleted?: ResponseStatus; + destIndexPatternDeleted?: ResponseStatus; + destinationIndex?: string | undefined; +} + +export interface DeleteTransformsResponseSchema { + [key: string]: DeleteTransformStatus; +} diff --git a/x-pack/plugins/transform/common/api_schemas/field_histograms.ts b/x-pack/plugins/transform/common/api_schemas/field_histograms.ts new file mode 100644 index 0000000000000..3bdbb5f1ff702 --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/field_histograms.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const fieldHistogramsRequestSchema = schema.object({ + /** Query to match documents in the index. */ + query: schema.any(), + /** The fields to return histogram data. */ + fields: schema.arrayOf(schema.any()), + /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ + samplerShardSize: schema.number(), +}); + +export type FieldHistogramsRequestSchema = TypeOf; +export type FieldHistogramsResponseSchema = any[]; diff --git a/x-pack/plugins/transform/common/api_schemas/start_transforms.ts b/x-pack/plugins/transform/common/api_schemas/start_transforms.ts new file mode 100644 index 0000000000000..b9611636e61a8 --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/start_transforms.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TypeOf } from '@kbn/config-schema'; + +import { transformIdsSchema, CommonResponseStatusSchema } from './common'; + +export const startTransformsRequestSchema = transformIdsSchema; +export type StartTransformsRequestSchema = TypeOf; +export type StartTransformsResponseSchema = CommonResponseStatusSchema; diff --git a/x-pack/plugins/transform/common/api_schemas/stop_transforms.ts b/x-pack/plugins/transform/common/api_schemas/stop_transforms.ts new file mode 100644 index 0000000000000..56956de20b49e --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/stop_transforms.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +import { transformStateSchema, CommonResponseStatusSchema } from './common'; + +export const stopTransformsRequestSchema = schema.arrayOf( + schema.object({ + id: schema.string(), + state: transformStateSchema, + }) +); + +export type StopTransformsRequestSchema = TypeOf; +export type StopTransformsResponseSchema = CommonResponseStatusSchema; diff --git a/x-pack/plugins/transform/common/api_schemas/transforms.ts b/x-pack/plugins/transform/common/api_schemas/transforms.ts new file mode 100644 index 0000000000000..155807a5c445f --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/transforms.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +import type { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common'; + +import type { Dictionary } from '../types/common'; +import type { PivotAggDict } from '../types/pivot_aggs'; +import type { PivotGroupByDict } from '../types/pivot_group_by'; +import type { TransformId, TransformPivotConfig } from '../types/transform'; + +import { transformStateSchema } from './common'; + +// GET transforms +export const getTransformsRequestSchema = schema.arrayOf( + schema.object({ + id: schema.string(), + state: transformStateSchema, + }) +); + +export type GetTransformsRequestSchema = TypeOf; + +export interface GetTransformsResponseSchema { + count: number; + transforms: TransformPivotConfig[]; +} + +// schemas shared by parts of the preview, create and update endpoint +export const destSchema = schema.object({ + index: schema.string(), + pipeline: schema.maybe(schema.string()), +}); +export const pivotSchema = schema.object({ + group_by: schema.any(), + aggregations: schema.any(), +}); +export const settingsSchema = schema.object({ + max_page_search_size: schema.maybe(schema.number()), + // The default value is null, which disables throttling. + docs_per_second: schema.maybe(schema.nullable(schema.number())), +}); +export const sourceSchema = schema.object({ + index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), + query: schema.maybe(schema.recordOf(schema.string(), schema.any())), +}); +export const syncSchema = schema.object({ + time: schema.object({ + delay: schema.maybe(schema.string()), + field: schema.string(), + }), +}); + +// PUT transforms/{transformId} +export const putTransformsRequestSchema = schema.object({ + description: schema.maybe(schema.string()), + dest: destSchema, + frequency: schema.maybe(schema.string()), + pivot: pivotSchema, + settings: schema.maybe(settingsSchema), + source: sourceSchema, + sync: schema.maybe(syncSchema), +}); + +export interface PutTransformsRequestSchema extends TypeOf { + pivot: { + group_by: PivotGroupByDict; + aggregations: PivotAggDict; + }; +} + +interface TransformCreated { + transform: TransformId; +} +interface TransformCreatedError { + id: TransformId; + error: any; +} +export interface PutTransformsResponseSchema { + transformsCreated: TransformCreated[]; + errors: TransformCreatedError[]; +} + +// POST transforms/_preview +export const postTransformsPreviewRequestSchema = schema.object({ + pivot: pivotSchema, + source: sourceSchema, +}); + +export interface PostTransformsPreviewRequestSchema + extends TypeOf { + pivot: { + group_by: PivotGroupByDict; + aggregations: PivotAggDict; + }; +} + +interface EsMappingType { + type: ES_FIELD_TYPES; +} + +export type PreviewItem = Dictionary; +export type PreviewData = PreviewItem[]; +export type PreviewMappingsProperties = Dictionary; + +export interface PostTransformsPreviewResponseSchema { + generated_dest_index: { + mappings: { + _meta: { + _transform: { + transform: string; + version: { create: string }; + creation_date_in_millis: number; + }; + created_by: string; + }; + properties: PreviewMappingsProperties; + }; + settings: { index: { number_of_shards: string; auto_expand_replicas: string } }; + aliases: Record; + }; + preview: PreviewData; +} diff --git a/x-pack/plugins/transform/common/api_schemas/transforms_stats.ts b/x-pack/plugins/transform/common/api_schemas/transforms_stats.ts new file mode 100644 index 0000000000000..30661a8a407da --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/transforms_stats.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TypeOf } from '@kbn/config-schema'; + +import { TransformStats } from '../types/transform_stats'; + +import { getTransformsRequestSchema } from './transforms'; + +export const getTransformsStatsRequestSchema = getTransformsRequestSchema; + +export type GetTransformsRequestSchema = TypeOf; + +export interface GetTransformsStatsResponseSchema { + node_failures?: object; + count: number; + transforms: TransformStats[]; +} diff --git a/x-pack/plugins/transform/common/api_schemas/type_guards.ts b/x-pack/plugins/transform/common/api_schemas/type_guards.ts new file mode 100644 index 0000000000000..f9753a412527e --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/type_guards.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import type { SearchResponse7 } from '../../../ml/common'; + +import type { EsIndex } from '../types/es_index'; + +// To be able to use the type guards on the client side, we need to make sure we don't import +// the code of '@kbn/config-schema' but just its types, otherwise the client side code will +// fail to build. +import type { FieldHistogramsResponseSchema } from './field_histograms'; +import type { GetTransformsAuditMessagesResponseSchema } from './audit_messages'; +import type { DeleteTransformsResponseSchema } from './delete_transforms'; +import type { StartTransformsResponseSchema } from './start_transforms'; +import type { StopTransformsResponseSchema } from './stop_transforms'; +import type { + GetTransformsResponseSchema, + PostTransformsPreviewResponseSchema, + PutTransformsResponseSchema, +} from './transforms'; +import type { GetTransformsStatsResponseSchema } from './transforms_stats'; +import type { PostTransformsUpdateResponseSchema } from './update_transforms'; + +const isBasicObject = (arg: any) => { + return typeof arg === 'object' && arg !== null; +}; + +const isGenericResponseSchema = (arg: any): arg is T => { + return ( + isBasicObject(arg) && + {}.hasOwnProperty.call(arg, 'count') && + {}.hasOwnProperty.call(arg, 'transforms') && + Array.isArray(arg.transforms) + ); +}; + +export const isGetTransformsResponseSchema = (arg: any): arg is GetTransformsResponseSchema => { + return isGenericResponseSchema(arg); +}; + +export const isGetTransformsStatsResponseSchema = ( + arg: any +): arg is GetTransformsStatsResponseSchema => { + return isGenericResponseSchema(arg); +}; + +export const isDeleteTransformsResponseSchema = ( + arg: any +): arg is DeleteTransformsResponseSchema => { + return ( + isBasicObject(arg) && + Object.values(arg).every((d) => ({}.hasOwnProperty.call(d, 'transformDeleted'))) + ); +}; + +export const isEsIndices = (arg: any): arg is EsIndex[] => { + return Array.isArray(arg); +}; + +export const isEsSearchResponse = (arg: any): arg is SearchResponse7 => { + return isBasicObject(arg) && {}.hasOwnProperty.call(arg, 'hits'); +}; + +export const isFieldHistogramsResponseSchema = (arg: any): arg is FieldHistogramsResponseSchema => { + return Array.isArray(arg); +}; + +export const isGetTransformsAuditMessagesResponseSchema = ( + arg: any +): arg is GetTransformsAuditMessagesResponseSchema => { + return Array.isArray(arg); +}; + +export const isPostTransformsPreviewResponseSchema = ( + arg: any +): arg is PostTransformsPreviewResponseSchema => { + return ( + isBasicObject(arg) && + {}.hasOwnProperty.call(arg, 'generated_dest_index') && + {}.hasOwnProperty.call(arg, 'preview') && + typeof arg.generated_dest_index !== undefined && + Array.isArray(arg.preview) + ); +}; + +export const isPostTransformsUpdateResponseSchema = ( + arg: any +): arg is PostTransformsUpdateResponseSchema => { + return isBasicObject(arg) && {}.hasOwnProperty.call(arg, 'id') && typeof arg.id === 'string'; +}; + +export const isPutTransformsResponseSchema = (arg: any): arg is PutTransformsResponseSchema => { + return ( + isBasicObject(arg) && + {}.hasOwnProperty.call(arg, 'transformsCreated') && + {}.hasOwnProperty.call(arg, 'errors') && + Array.isArray(arg.transformsCreated) && + Array.isArray(arg.errors) + ); +}; + +const isGenericSuccessResponseSchema = (arg: any) => + isBasicObject(arg) && Object.values(arg).every((d) => ({}.hasOwnProperty.call(d, 'success'))); + +export const isStartTransformsResponseSchema = (arg: any): arg is StartTransformsResponseSchema => { + return isGenericSuccessResponseSchema(arg); +}; + +export const isStopTransformsResponseSchema = (arg: any): arg is StopTransformsResponseSchema => { + return isGenericSuccessResponseSchema(arg); +}; diff --git a/x-pack/plugins/transform/common/api_schemas/update_transforms.ts b/x-pack/plugins/transform/common/api_schemas/update_transforms.ts new file mode 100644 index 0000000000000..e303d94ef0536 --- /dev/null +++ b/x-pack/plugins/transform/common/api_schemas/update_transforms.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +import { TransformPivotConfig } from '../types/transform'; + +import { destSchema, settingsSchema, sourceSchema, syncSchema } from './transforms'; + +// POST _transform/{transform_id}/_update +export const postTransformsUpdateRequestSchema = schema.object({ + description: schema.maybe(schema.string()), + dest: schema.maybe(destSchema), + frequency: schema.maybe(schema.string()), + settings: schema.maybe(settingsSchema), + source: schema.maybe(sourceSchema), + sync: schema.maybe(syncSchema), +}); + +export type PostTransformsUpdateRequestSchema = TypeOf; +export type PostTransformsUpdateResponseSchema = TransformPivotConfig; diff --git a/x-pack/plugins/transform/common/constants.ts b/x-pack/plugins/transform/common/constants.ts index b01a82dffa04a..5efb6f31c1e3f 100644 --- a/x-pack/plugins/transform/common/constants.ts +++ b/x-pack/plugins/transform/common/constants.ts @@ -75,3 +75,24 @@ export const APP_CREATE_TRANSFORM_CLUSTER_PRIVILEGES = [ ]; export const APP_INDEX_PRIVILEGES = ['monitor']; + +// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243 +export const TRANSFORM_STATE = { + ABORTING: 'aborting', + FAILED: 'failed', + INDEXING: 'indexing', + STARTED: 'started', + STOPPED: 'stopped', + STOPPING: 'stopping', +} as const; + +const transformStates = Object.values(TRANSFORM_STATE); +export type TransformState = typeof transformStates[number]; + +export const TRANSFORM_MODE = { + BATCH: 'batch', + CONTINUOUS: 'continuous', +} as const; + +const transformModes = Object.values(TRANSFORM_MODE); +export type TransformMode = typeof transformModes[number]; diff --git a/x-pack/plugins/transform/common/index.ts b/x-pack/plugins/transform/common/index.ts deleted file mode 100644 index 08bb4022c7016..0000000000000 --- a/x-pack/plugins/transform/common/index.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export interface MissingPrivileges { - [key: string]: string[] | undefined; -} - -export interface Privileges { - hasAllPrivileges: boolean; - missingPrivileges: MissingPrivileges; -} - -export type TransformId = string; - -// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243 -export enum TRANSFORM_STATE { - ABORTING = 'aborting', - FAILED = 'failed', - INDEXING = 'indexing', - STARTED = 'started', - STOPPED = 'stopped', - STOPPING = 'stopping', -} - -export interface TransformEndpointRequest { - id: TransformId; - state?: TRANSFORM_STATE; -} - -export interface ResultData { - success: boolean; - error?: any; -} - -export interface TransformEndpointResult { - [key: string]: ResultData; -} - -export interface DeleteTransformEndpointRequest { - transformsInfo: TransformEndpointRequest[]; - deleteDestIndex?: boolean; - deleteDestIndexPattern?: boolean; - forceDelete?: boolean; -} - -export interface DeleteTransformStatus { - transformDeleted: ResultData; - destIndexDeleted?: ResultData; - destIndexPatternDeleted?: ResultData; - destinationIndex?: string | undefined; -} - -export interface DeleteTransformEndpointResult { - [key: string]: DeleteTransformStatus; -} diff --git a/x-pack/plugins/transform/common/shared_imports.ts b/x-pack/plugins/transform/common/shared_imports.ts new file mode 100644 index 0000000000000..8681204755c36 --- /dev/null +++ b/x-pack/plugins/transform/common/shared_imports.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type { SearchResponse7 } from '../../ml/common'; diff --git a/x-pack/plugins/transform/common/types/aggregations.ts b/x-pack/plugins/transform/common/types/aggregations.ts new file mode 100644 index 0000000000000..77b7e55e3ba94 --- /dev/null +++ b/x-pack/plugins/transform/common/types/aggregations.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type AggName = string; diff --git a/x-pack/plugins/transform/public/app/hooks/use_api_types.ts b/x-pack/plugins/transform/common/types/es_index.ts similarity index 100% rename from x-pack/plugins/transform/public/app/hooks/use_api_types.ts rename to x-pack/plugins/transform/common/types/es_index.ts diff --git a/x-pack/plugins/transform/common/types/fields.ts b/x-pack/plugins/transform/common/types/fields.ts new file mode 100644 index 0000000000000..2c274f3bd9b48 --- /dev/null +++ b/x-pack/plugins/transform/common/types/fields.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export type EsFieldName = string; diff --git a/x-pack/plugins/transform/common/types/pivot_aggs.ts b/x-pack/plugins/transform/common/types/pivot_aggs.ts new file mode 100644 index 0000000000000..d50609da6a5dc --- /dev/null +++ b/x-pack/plugins/transform/common/types/pivot_aggs.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AggName } from './aggregations'; +import { EsFieldName } from './fields'; + +export const PIVOT_SUPPORTED_AGGS = { + AVG: 'avg', + CARDINALITY: 'cardinality', + MAX: 'max', + MIN: 'min', + PERCENTILES: 'percentiles', + SUM: 'sum', + VALUE_COUNT: 'value_count', + FILTER: 'filter', +} as const; + +export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS]; + +export type PivotAgg = { + [key in PivotSupportedAggs]?: { + field: EsFieldName; + }; +}; + +export type PivotAggDict = { + [key in AggName]: PivotAgg; +}; diff --git a/x-pack/plugins/transform/common/types/pivot_group_by.ts b/x-pack/plugins/transform/common/types/pivot_group_by.ts new file mode 100644 index 0000000000000..bfaf17a32b580 --- /dev/null +++ b/x-pack/plugins/transform/common/types/pivot_group_by.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Dictionary } from './common'; +import { EsFieldName } from './fields'; + +export type GenericAgg = object; + +export interface TermsAgg { + terms: { + field: EsFieldName; + }; +} + +export interface HistogramAgg { + histogram: { + field: EsFieldName; + interval: string; + }; +} + +export interface DateHistogramAgg { + date_histogram: { + field: EsFieldName; + calendar_interval: string; + }; +} + +export type PivotGroupBy = GenericAgg | TermsAgg | HistogramAgg | DateHistogramAgg; +export type PivotGroupByDict = Dictionary; diff --git a/x-pack/plugins/transform/common/types/privileges.ts b/x-pack/plugins/transform/common/types/privileges.ts new file mode 100644 index 0000000000000..bf710b8225599 --- /dev/null +++ b/x-pack/plugins/transform/common/types/privileges.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface MissingPrivileges { + [key: string]: string[] | undefined; +} + +export interface Privileges { + hasAllPrivileges: boolean; + missingPrivileges: MissingPrivileges; +} diff --git a/x-pack/plugins/transform/common/types/transform.ts b/x-pack/plugins/transform/common/types/transform.ts new file mode 100644 index 0000000000000..6b31705442706 --- /dev/null +++ b/x-pack/plugins/transform/common/types/transform.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import type { PutTransformsRequestSchema } from '../api_schemas/transforms'; + +export type IndexName = string; +export type IndexPattern = string; +export type TransformId = string; + +export interface TransformPivotConfig extends PutTransformsRequestSchema { + id: TransformId; + create_time?: number; + version?: string; +} diff --git a/x-pack/plugins/transform/common/types/transform_stats.ts b/x-pack/plugins/transform/common/types/transform_stats.ts new file mode 100644 index 0000000000000..5bd2fd955845c --- /dev/null +++ b/x-pack/plugins/transform/common/types/transform_stats.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TransformState, TRANSFORM_STATE } from '../constants'; +import { TransformId } from './transform'; + +export interface TransformStats { + id: TransformId; + checkpointing: { + last: { + checkpoint: number; + timestamp_millis?: number; + }; + next?: { + checkpoint: number; + checkpoint_progress?: { + total_docs: number; + docs_remaining: number; + percent_complete: number; + }; + }; + operations_behind: number; + }; + node?: { + id: string; + name: string; + ephemeral_id: string; + transport_address: string; + attributes: Record; + }; + stats: { + documents_indexed: number; + documents_processed: number; + index_failures: number; + index_time_in_ms: number; + index_total: number; + pages_processed: number; + search_failures: number; + search_time_in_ms: number; + search_total: number; + trigger_count: number; + processing_time_in_ms: number; + processing_total: number; + exponential_avg_checkpoint_duration_ms: number; + exponential_avg_documents_indexed: number; + exponential_avg_documents_processed: number; + }; + reason?: string; + state: TransformState; +} + +export function isTransformStats(arg: any): arg is TransformStats { + return ( + typeof arg === 'object' && + arg !== null && + {}.hasOwnProperty.call(arg, 'state') && + Object.values(TRANSFORM_STATE).includes(arg.state) + ); +} diff --git a/x-pack/plugins/transform/public/__mocks__/shared_imports.ts b/x-pack/plugins/transform/public/__mocks__/shared_imports.ts index f7441fd93f38a..470c42d5de7fa 100644 --- a/x-pack/plugins/transform/public/__mocks__/shared_imports.ts +++ b/x-pack/plugins/transform/public/__mocks__/shared_imports.ts @@ -23,7 +23,6 @@ export { DataGrid, EsSorting, RenderCellValue, - SearchResponse7, UseDataGridReturnType, UseIndexDataReturnType, INDEX_STATUS, diff --git a/x-pack/plugins/transform/public/app/common/aggregations.ts b/x-pack/plugins/transform/public/app/common/aggregations.ts index 397a58006f1d1..507579d374353 100644 --- a/x-pack/plugins/transform/public/app/common/aggregations.ts +++ b/x-pack/plugins/transform/public/app/common/aggregations.ts @@ -6,7 +6,7 @@ import { composeValidators, patternValidator } from '../../../../ml/public'; -export type AggName = string; +import { AggName } from '../../../common/types/aggregations'; export function isAggName(arg: any): arg is AggName { // allow all characters except `[]>` and must not start or end with a space. diff --git a/x-pack/plugins/transform/public/app/common/data_grid.test.ts b/x-pack/plugins/transform/public/app/common/data_grid.test.ts index 0e5ecb5d3b214..6d96f614b28a4 100644 --- a/x-pack/plugins/transform/public/app/common/data_grid.test.ts +++ b/x-pack/plugins/transform/public/app/common/data_grid.test.ts @@ -4,11 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +import { PIVOT_SUPPORTED_AGGS } from '../../../common/types/pivot_aggs'; + import { - getPreviewRequestBody, + getPreviewTransformRequestBody, PivotAggsConfig, PivotGroupByConfig, - PIVOT_SUPPORTED_AGGS, PIVOT_SUPPORTED_GROUP_BY_AGGS, SimpleQuery, } from '../common'; @@ -35,7 +36,12 @@ describe('Transform: Data Grid', () => { aggName: 'the-agg-agg-name', dropDownName: 'the-agg-drop-down-name', }; - const request = getPreviewRequestBody('the-index-pattern-title', query, [groupBy], [agg]); + const request = getPreviewTransformRequestBody( + 'the-index-pattern-title', + query, + [groupBy], + [agg] + ); const pivotPreviewDevConsoleStatement = getPivotPreviewDevConsoleStatement(request); expect(pivotPreviewDevConsoleStatement).toBe(`POST _transform/_preview diff --git a/x-pack/plugins/transform/public/app/common/data_grid.ts b/x-pack/plugins/transform/public/app/common/data_grid.ts index cf9ba5d6f5853..08f834431fa8b 100644 --- a/x-pack/plugins/transform/public/app/common/data_grid.ts +++ b/x-pack/plugins/transform/public/app/common/data_grid.ts @@ -4,12 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ +import type { PostTransformsPreviewRequestSchema } from '../../../common/api_schemas/transforms'; + import { PivotQuery } from './request'; -import { PreviewRequestBody } from './transform'; export const INIT_MAX_COLUMNS = 20; -export const getPivotPreviewDevConsoleStatement = (request: PreviewRequestBody) => { +export const getPivotPreviewDevConsoleStatement = (request: PostTransformsPreviewRequestSchema) => { return `POST _transform/_preview\n${JSON.stringify(request, null, 2)}\n`; }; diff --git a/x-pack/plugins/transform/public/app/common/fields.ts b/x-pack/plugins/transform/public/app/common/fields.ts index b22aae255b9fa..778750e1f97e4 100644 --- a/x-pack/plugins/transform/public/app/common/fields.ts +++ b/x-pack/plugins/transform/public/app/common/fields.ts @@ -5,10 +5,10 @@ */ import { Dictionary } from '../../../common/types/common'; +import { EsFieldName } from '../../../common/types/fields'; export type EsId = string; export type EsDocSource = Dictionary; -export type EsFieldName = string; export interface EsDoc extends Dictionary { _id: EsId; diff --git a/x-pack/plugins/transform/public/app/common/index.ts b/x-pack/plugins/transform/public/app/common/index.ts index 45ddc440057b2..0fc947eaf33b0 100644 --- a/x-pack/plugins/transform/public/app/common/index.ts +++ b/x-pack/plugins/transform/public/app/common/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export { AggName, isAggName } from './aggregations'; +export { isAggName } from './aggregations'; export { getIndexDevConsoleStatement, getPivotPreviewDevConsoleStatement, @@ -17,44 +17,28 @@ export { toggleSelectedField, EsDoc, EsDocSource, - EsFieldName, } from './fields'; export { DropDownLabel, DropDownOption, Label } from './dropdown'; export { isTransformIdValid, refreshTransformList$, useRefreshTransformList, - CreateRequestBody, - PreviewRequestBody, - TransformPivotConfig, - IndexName, - IndexPattern, REFRESH_TRANSFORM_LIST_STATE, } from './transform'; export { TRANSFORM_LIST_COLUMN, TransformListAction, TransformListRow } from './transform_list'; -export { - getTransformProgress, - isCompletedBatchTransform, - isTransformStats, - TransformStats, - TRANSFORM_MODE, -} from './transform_stats'; +export { getTransformProgress, isCompletedBatchTransform } from './transform_stats'; export { getDiscoverUrl } from './navigation'; -export { GetTransformsResponse, PreviewData, PreviewMappings } from './pivot_preview'; export { getEsAggFromAggConfig, isPivotAggsConfigWithUiSupport, isPivotAggsConfigPercentiles, PERCENTILES_AGG_DEFAULT_PERCENTS, - PivotAgg, - PivotAggDict, PivotAggsConfig, PivotAggsConfigDict, PivotAggsConfigBase, PivotAggsConfigWithUiSupport, PivotAggsConfigWithUiSupportDict, pivotAggsFieldSupport, - PIVOT_SUPPORTED_AGGS, } from './pivot_aggs'; export { dateHistogramIntervalFormatRegex, @@ -65,25 +49,19 @@ export { isGroupByHistogram, isGroupByTerms, pivotGroupByFieldSupport, - DateHistogramAgg, - GenericAgg, GroupByConfigWithInterval, GroupByConfigWithUiSupport, - HistogramAgg, - PivotGroupBy, PivotGroupByConfig, - PivotGroupByDict, PivotGroupByConfigDict, PivotGroupByConfigWithUiSupportDict, PivotSupportedGroupByAggs, PivotSupportedGroupByAggsWithInterval, PIVOT_SUPPORTED_GROUP_BY_AGGS, - TermsAgg, } from './pivot_group_by'; export { defaultQuery, - getPreviewRequestBody, - getCreateRequestBody, + getPreviewTransformRequestBody, + getCreateTransformRequestBody, getPivotQuery, isDefaultQuery, isMatchAllQuery, diff --git a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts index ec52de4b9da92..7a7bb4c65b306 100644 --- a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts +++ b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts @@ -5,31 +5,22 @@ */ import { FC } from 'react'; -import { Dictionary } from '../../../common/types/common'; + import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; -import { AggName } from './aggregations'; -import { EsFieldName } from './fields'; +import type { AggName } from '../../../common/types/aggregations'; +import type { Dictionary } from '../../../common/types/common'; +import type { EsFieldName } from '../../../common/types/fields'; +import type { PivotAgg, PivotSupportedAggs } from '../../../common/types/pivot_aggs'; +import { PIVOT_SUPPORTED_AGGS } from '../../../common/types/pivot_aggs'; + import { getAggFormConfig } from '../sections/create_transform/components/step_define/common/get_agg_form_config'; import { PivotAggsConfigFilter } from '../sections/create_transform/components/step_define/common/filter_agg/types'; -export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS]; - export function isPivotSupportedAggs(arg: any): arg is PivotSupportedAggs { return Object.values(PIVOT_SUPPORTED_AGGS).includes(arg); } -export const PIVOT_SUPPORTED_AGGS = { - AVG: 'avg', - CARDINALITY: 'cardinality', - MAX: 'max', - MIN: 'min', - PERCENTILES: 'percentiles', - SUM: 'sum', - VALUE_COUNT: 'value_count', - FILTER: 'filter', -} as const; - export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99]; export const pivotAggsFieldSupport = { @@ -69,16 +60,6 @@ export const pivotAggsFieldSupport = { [KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], }; -export type PivotAgg = { - [key in PivotSupportedAggs]?: { - field: EsFieldName; - }; -}; - -export type PivotAggDict = { - [key in AggName]: PivotAgg; -}; - /** * The maximum level of sub-aggregations */ diff --git a/x-pack/plugins/transform/public/app/common/pivot_group_by.ts b/x-pack/plugins/transform/public/app/common/pivot_group_by.ts index 7da52fc018338..2c2bac369c72d 100644 --- a/x-pack/plugins/transform/public/app/common/pivot_group_by.ts +++ b/x-pack/plugins/transform/public/app/common/pivot_group_by.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +import { AggName } from '../../../common/types/aggregations'; import { Dictionary } from '../../../common/types/common'; +import { EsFieldName } from '../../../common/types/fields'; +import { GenericAgg } from '../../../common/types/pivot_group_by'; import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; -import { AggName } from './aggregations'; -import { EsFieldName } from './fields'; - export enum PIVOT_SUPPORTED_GROUP_BY_AGGS { DATE_HISTOGRAM = 'date_histogram', HISTOGRAM = 'histogram', @@ -106,31 +106,6 @@ export function isPivotGroupByConfigWithUiSupport(arg: any): arg is GroupByConfi return isGroupByDateHistogram(arg) || isGroupByHistogram(arg) || isGroupByTerms(arg); } -export type GenericAgg = object; - -export interface TermsAgg { - terms: { - field: EsFieldName; - }; -} - -export interface HistogramAgg { - histogram: { - field: EsFieldName; - interval: string; - }; -} - -export interface DateHistogramAgg { - date_histogram: { - field: EsFieldName; - calendar_interval: string; - }; -} - -export type PivotGroupBy = GenericAgg | TermsAgg | HistogramAgg | DateHistogramAgg; -export type PivotGroupByDict = Dictionary; - export function getEsAggFromGroupByConfig(groupByConfig: GroupByConfigBase): GenericAgg { const { agg, aggName, dropDownName, ...esAgg } = groupByConfig; diff --git a/x-pack/plugins/transform/public/app/common/pivot_preview.ts b/x-pack/plugins/transform/public/app/common/pivot_preview.ts deleted file mode 100644 index 14368a80b0131..0000000000000 --- a/x-pack/plugins/transform/public/app/common/pivot_preview.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; - -import { Dictionary } from '../../../common/types/common'; - -interface EsMappingType { - type: ES_FIELD_TYPES; -} - -export type PreviewItem = Dictionary; -export type PreviewData = PreviewItem[]; -export interface PreviewMappings { - properties: Dictionary; -} - -export interface GetTransformsResponse { - preview: PreviewData; - generated_dest_index: { - mappings: PreviewMappings; - // Not in use yet - aliases: any; - settings: any; - }; -} diff --git a/x-pack/plugins/transform/public/app/common/request.test.ts b/x-pack/plugins/transform/public/app/common/request.test.ts index 63f1f8b10ad44..416927c460842 100644 --- a/x-pack/plugins/transform/public/app/common/request.test.ts +++ b/x-pack/plugins/transform/public/app/common/request.test.ts @@ -4,17 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ +import { PIVOT_SUPPORTED_AGGS } from '../../../common/types/pivot_aggs'; + import { PivotGroupByConfig } from '../common'; import { StepDefineExposedState } from '../sections/create_transform/components/step_define'; import { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form'; import { PIVOT_SUPPORTED_GROUP_BY_AGGS } from './pivot_group_by'; -import { PivotAggsConfig, PIVOT_SUPPORTED_AGGS } from './pivot_aggs'; +import { PivotAggsConfig } from './pivot_aggs'; import { defaultQuery, - getPreviewRequestBody, - getCreateRequestBody, + getPreviewTransformRequestBody, + getCreateTransformRequestBody, getPivotQuery, isDefaultQuery, isMatchAllQuery, @@ -55,7 +57,7 @@ describe('Transform: Common', () => { }); }); - test('getPreviewRequestBody()', () => { + test('getPreviewTransformRequestBody()', () => { const query = getPivotQuery('the-query'); const groupBy: PivotGroupByConfig[] = [ { @@ -73,7 +75,7 @@ describe('Transform: Common', () => { dropDownName: 'the-agg-drop-down-name', }, ]; - const request = getPreviewRequestBody('the-index-pattern-title', query, groupBy, aggs); + const request = getPreviewTransformRequestBody('the-index-pattern-title', query, groupBy, aggs); expect(request).toEqual({ pivot: { @@ -87,7 +89,7 @@ describe('Transform: Common', () => { }); }); - test('getPreviewRequestBody() with comma-separated index pattern', () => { + test('getPreviewTransformRequestBody() with comma-separated index pattern', () => { const query = getPivotQuery('the-query'); const groupBy: PivotGroupByConfig[] = [ { @@ -105,7 +107,7 @@ describe('Transform: Common', () => { dropDownName: 'the-agg-drop-down-name', }, ]; - const request = getPreviewRequestBody( + const request = getPreviewTransformRequestBody( 'the-index-pattern-title,the-other-title', query, groupBy, @@ -124,7 +126,7 @@ describe('Transform: Common', () => { }); }); - test('getCreateRequestBody()', () => { + test('getCreateTransformRequestBody()', () => { const groupBy: PivotGroupByConfig = { agg: PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS, field: 'the-group-by-field', @@ -160,7 +162,7 @@ describe('Transform: Common', () => { valid: true, }; - const request = getCreateRequestBody( + const request = getCreateTransformRequestBody( 'the-index-pattern-title', pivotState, transformDetailsState diff --git a/x-pack/plugins/transform/public/app/common/request.ts b/x-pack/plugins/transform/public/app/common/request.ts index 9a0084c2ebffb..10f3a63477029 100644 --- a/x-pack/plugins/transform/public/app/common/request.ts +++ b/x-pack/plugins/transform/public/app/common/request.ts @@ -4,15 +4,25 @@ * you may not use this file except in compliance with the Elastic License. */ -import { DefaultOperator } from 'elasticsearch'; - +import type { DefaultOperator } from 'elasticsearch'; + +import { HttpFetchError } from '../../../../../../src/core/public'; +import type { IndexPattern } from '../../../../../../src/plugins/data/public'; + +import type { + PostTransformsPreviewRequestSchema, + PutTransformsRequestSchema, +} from '../../../common/api_schemas/transforms'; +import type { + DateHistogramAgg, + HistogramAgg, + TermsAgg, +} from '../../../common/types/pivot_group_by'; import { dictionaryToArray } from '../../../common/types/common'; -import { SavedSearchQuery } from '../hooks/use_search_items'; - -import { StepDefineExposedState } from '../sections/create_transform/components/step_define'; -import { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form'; -import { IndexPattern } from '../../../../../../src/plugins/data/public'; +import type { SavedSearchQuery } from '../hooks/use_search_items'; +import type { StepDefineExposedState } from '../sections/create_transform/components/step_define'; +import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form'; import { getEsAggFromAggConfig, @@ -24,8 +34,6 @@ import { } from '../common'; import { PivotAggsConfig } from './pivot_aggs'; -import { DateHistogramAgg, HistogramAgg, TermsAgg } from './pivot_group_by'; -import { PreviewRequestBody, CreateRequestBody } from './transform'; export interface SimpleQuery { query_string: { @@ -63,17 +71,18 @@ export function isDefaultQuery(query: PivotQuery): boolean { return isSimpleQuery(query) && query.query_string.query === '*'; } -export function getPreviewRequestBody( +export function getPreviewTransformRequestBody( indexPatternTitle: IndexPattern['title'], query: PivotQuery, groupBy: PivotGroupByConfig[], aggs: PivotAggsConfig[] -): PreviewRequestBody { +): PostTransformsPreviewRequestSchema { const index = indexPatternTitle.split(',').map((name: string) => name.trim()); - const request: PreviewRequestBody = { + const request: PostTransformsPreviewRequestSchema = { source: { index, + ...(!isDefaultQuery(query) && !isMatchAllQuery(query) ? { query } : {}), }, pivot: { group_by: {}, @@ -81,10 +90,6 @@ export function getPreviewRequestBody( }, }; - if (!isDefaultQuery(query) && !isMatchAllQuery(query)) { - request.source.query = query; - } - groupBy.forEach((g) => { if (isGroupByTerms(g)) { const termsAgg: TermsAgg = { @@ -125,37 +130,41 @@ export function getPreviewRequestBody( return request; } -export function getCreateRequestBody( +export const getCreateTransformRequestBody = ( indexPatternTitle: IndexPattern['title'], pivotState: StepDefineExposedState, transformDetailsState: StepDetailsExposedState -): CreateRequestBody { - const request: CreateRequestBody = { - ...getPreviewRequestBody( - indexPatternTitle, - getPivotQuery(pivotState.searchQuery), - dictionaryToArray(pivotState.groupByList), - dictionaryToArray(pivotState.aggList) - ), - // conditionally add optional description - ...(transformDetailsState.transformDescription !== '' - ? { description: transformDetailsState.transformDescription } - : {}), - dest: { - index: transformDetailsState.destinationIndex, - }, - // conditionally add continuous mode config - ...(transformDetailsState.isContinuousModeEnabled - ? { - sync: { - time: { - field: transformDetailsState.continuousModeDateField, - delay: transformDetailsState.continuousModeDelay, - }, +): PutTransformsRequestSchema => ({ + ...getPreviewTransformRequestBody( + indexPatternTitle, + getPivotQuery(pivotState.searchQuery), + dictionaryToArray(pivotState.groupByList), + dictionaryToArray(pivotState.aggList) + ), + // conditionally add optional description + ...(transformDetailsState.transformDescription !== '' + ? { description: transformDetailsState.transformDescription } + : {}), + dest: { + index: transformDetailsState.destinationIndex, + }, + // conditionally add continuous mode config + ...(transformDetailsState.isContinuousModeEnabled + ? { + sync: { + time: { + field: transformDetailsState.continuousModeDateField, + delay: transformDetailsState.continuousModeDelay, }, - } - : {}), - }; - - return request; + }, + } + : {}), +}); + +export function isHttpFetchError(error: any): error is HttpFetchError { + return ( + error instanceof HttpFetchError && + typeof error.name === 'string' && + typeof error.message !== 'undefined' + ); } diff --git a/x-pack/plugins/transform/public/app/common/transform.ts b/x-pack/plugins/transform/public/app/common/transform.ts index a02bed2fa65e7..b71bab62096b6 100644 --- a/x-pack/plugins/transform/public/app/common/transform.ts +++ b/x-pack/plugins/transform/public/app/common/transform.ts @@ -9,13 +9,7 @@ import { BehaviorSubject } from 'rxjs'; import { filter, distinctUntilChanged } from 'rxjs/operators'; import { Subscription } from 'rxjs'; -import { TransformId } from '../../../common'; - -import { PivotAggDict } from './pivot_aggs'; -import { PivotGroupByDict } from './pivot_group_by'; - -export type IndexName = string; -export type IndexPattern = string; +import { TransformId } from '../../../common/types/transform'; // Transform name must contain lowercase alphanumeric (a-z and 0-9), hyphens or underscores; // It must also start and end with an alphanumeric character. @@ -23,41 +17,6 @@ export function isTransformIdValid(transformId: TransformId) { return /^[a-z0-9\-\_]+$/g.test(transformId) && !/^([_-].*)?(.*[_-])?$/g.test(transformId); } -export interface PreviewRequestBody { - pivot: { - group_by: PivotGroupByDict; - aggregations: PivotAggDict; - }; - source: { - index: IndexPattern | IndexPattern[]; - query?: any; - }; -} - -export interface CreateRequestBody extends PreviewRequestBody { - description?: string; - dest: { - index: IndexName; - }; - frequency?: string; - settings?: { - max_page_search_size?: number; - docs_per_second?: number; - }; - sync?: { - time: { - field: string; - delay: string; - }; - }; -} - -export interface TransformPivotConfig extends CreateRequestBody { - id: TransformId; - create_time?: number; - version?: string; -} - export enum REFRESH_TRANSFORM_LIST_STATE { ERROR = 'error', IDLE = 'idle', diff --git a/x-pack/plugins/transform/public/app/common/transform_list.ts b/x-pack/plugins/transform/public/app/common/transform_list.ts index a2a762a7e2dfb..b32803fea1501 100644 --- a/x-pack/plugins/transform/public/app/common/transform_list.ts +++ b/x-pack/plugins/transform/public/app/common/transform_list.ts @@ -6,9 +6,8 @@ import { EuiTableActionsColumnType } from '@elastic/eui'; -import { TransformId } from '../../../common'; -import { TransformPivotConfig } from './transform'; -import { TransformStats } from './transform_stats'; +import { TransformId, TransformPivotConfig } from '../../../common/types/transform'; +import { TransformStats } from '../../../common/types/transform_stats'; // Used to pass on attribute names to table columns export enum TRANSFORM_LIST_COLUMN { diff --git a/x-pack/plugins/transform/public/app/common/transform_stats.ts b/x-pack/plugins/transform/public/app/common/transform_stats.ts index 72df6d3985e23..aaf7f97399d44 100644 --- a/x-pack/plugins/transform/public/app/common/transform_stats.ts +++ b/x-pack/plugins/transform/public/app/common/transform_stats.ts @@ -4,64 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { TransformId, TRANSFORM_STATE } from '../../../common'; +import { TRANSFORM_STATE } from '../../../common/constants'; import { TransformListRow } from './transform_list'; -export enum TRANSFORM_MODE { - BATCH = 'batch', - CONTINUOUS = 'continuous', -} - -export interface TransformStats { - id: TransformId; - checkpointing: { - last: { - checkpoint: number; - timestamp_millis?: number; - }; - next?: { - checkpoint: number; - checkpoint_progress?: { - total_docs: number; - docs_remaining: number; - percent_complete: number; - }; - }; - operations_behind: number; - }; - node?: { - id: string; - name: string; - ephemeral_id: string; - transport_address: string; - attributes: Record; - }; - stats: { - documents_indexed: number; - documents_processed: number; - index_failures: number; - index_time_in_ms: number; - index_total: number; - pages_processed: number; - search_failures: number; - search_time_in_ms: number; - search_total: number; - trigger_count: number; - }; - reason?: string; - state: TRANSFORM_STATE; -} - -export function isTransformStats(arg: any): arg is TransformStats { - return ( - typeof arg === 'object' && - arg !== null && - {}.hasOwnProperty.call(arg, 'state') && - Object.values(TRANSFORM_STATE).includes(arg.state) - ); -} - export function getTransformProgress(item: TransformListRow) { if (isCompletedBatchTransform(item)) { return 100; diff --git a/x-pack/plugins/transform/public/app/hooks/__mocks__/use_api.ts b/x-pack/plugins/transform/public/app/hooks/__mocks__/use_api.ts index a5cccd58211c5..40a6ab2b65862 100644 --- a/x-pack/plugins/transform/public/app/hooks/__mocks__/use_api.ts +++ b/x-pack/plugins/transform/public/app/hooks/__mocks__/use_api.ts @@ -4,67 +4,162 @@ * you may not use this file except in compliance with the Elastic License. */ -import { TransformId, TransformEndpointRequest } from '../../../../common'; +import { HttpFetchError } from 'kibana/public'; -import { PreviewRequestBody } from '../../common'; +import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; + +import { TransformId } from '../../../../common/types/transform'; +import type { FieldHistogramsResponseSchema } from '../../../../common/api_schemas/field_histograms'; +import type { GetTransformsAuditMessagesResponseSchema } from '../../../../common/api_schemas/audit_messages'; +import type { + DeleteTransformsRequestSchema, + DeleteTransformsResponseSchema, +} from '../../../../common/api_schemas/delete_transforms'; +import type { + StartTransformsRequestSchema, + StartTransformsResponseSchema, +} from '../../../../common/api_schemas/start_transforms'; +import type { + StopTransformsRequestSchema, + StopTransformsResponseSchema, +} from '../../../../common/api_schemas/stop_transforms'; +import type { + GetTransformsResponseSchema, + PostTransformsPreviewRequestSchema, + PostTransformsPreviewResponseSchema, + PutTransformsRequestSchema, + PutTransformsResponseSchema, +} from '../../../../common/api_schemas/transforms'; +import type { GetTransformsStatsResponseSchema } from '../../../../common/api_schemas/transforms_stats'; +import type { + PostTransformsUpdateRequestSchema, + PostTransformsUpdateResponseSchema, +} from '../../../../common/api_schemas/update_transforms'; + +import type { SearchResponse7 } from '../../../../common/shared_imports'; +import { EsIndex } from '../../../../common/types/es_index'; + +import type { SavedSearchQuery } from '../use_search_items'; + +// Default sampler shard size used for field histograms +export const DEFAULT_SAMPLER_SHARD_SIZE = 5000; + +export interface FieldHistogramRequestConfig { + fieldName: string; + type?: KBN_FIELD_TYPES; +} const apiFactory = () => ({ - getTransforms(transformId?: TransformId): Promise { - return new Promise((resolve, reject) => { - resolve([]); - }); + async getTransform( + transformId: TransformId + ): Promise { + return Promise.resolve({ count: 0, transforms: [] }); }, - getTransformsStats(transformId?: TransformId): Promise { - if (transformId !== undefined) { - return new Promise((resolve, reject) => { - resolve([]); - }); - } - - return new Promise((resolve, reject) => { - resolve([]); - }); + async getTransforms(): Promise { + return Promise.resolve({ count: 0, transforms: [] }); }, - createTransform(transformId: TransformId, transformConfig: any): Promise { - return new Promise((resolve, reject) => { - resolve([]); - }); + async getTransformStats( + transformId: TransformId + ): Promise { + return Promise.resolve({ count: 0, transforms: [] }); }, - deleteTransforms(transformsInfo: TransformEndpointRequest[]) { - return new Promise((resolve, reject) => { - resolve([]); - }); + async getTransformsStats(): Promise { + return Promise.resolve({ count: 0, transforms: [] }); }, - getTransformsPreview(obj: PreviewRequestBody): Promise { - return new Promise((resolve, reject) => { - resolve([]); - }); + async createTransform( + transformId: TransformId, + transformConfig: PutTransformsRequestSchema + ): Promise { + return Promise.resolve({ transformsCreated: [], errors: [] }); }, - startTransforms(transformsInfo: TransformEndpointRequest[]) { - return new Promise((resolve, reject) => { - resolve([]); + async updateTransform( + transformId: TransformId, + transformConfig: PostTransformsUpdateRequestSchema + ): Promise { + return Promise.resolve({ + id: 'the-test-id', + source: { index: ['the-index-name'], query: { match_all: {} } }, + dest: { index: 'user-the-destination-index-name' }, + frequency: '10m', + pivot: { + group_by: { the_group: { terms: { field: 'the-group-by-field' } } }, + aggregations: { the_agg: { value_count: { field: 'the-agg-field' } } }, + }, + description: 'the-description', + settings: { docs_per_second: null }, + version: '8.0.0', + create_time: 1598860879097, }); }, - stopTransforms(transformsInfo: TransformEndpointRequest[]) { - return new Promise((resolve, reject) => { - resolve([]); - }); + async deleteTransforms( + reqBody: DeleteTransformsRequestSchema + ): Promise { + return Promise.resolve({}); }, - getTransformAuditMessages(transformId: TransformId): Promise { - return new Promise((resolve, reject) => { - resolve([]); + async getTransformsPreview( + obj: PostTransformsPreviewRequestSchema + ): Promise { + return Promise.resolve({ + generated_dest_index: { + mappings: { + _meta: { + _transform: { + transform: 'the-transform', + version: { create: 'the-version' }, + creation_date_in_millis: 0, + }, + created_by: 'mock', + }, + properties: {}, + }, + settings: { index: { number_of_shards: '1', auto_expand_replicas: '0-1' } }, + aliases: {}, + }, + preview: [], }); }, - esSearch(payload: any) { - return new Promise((resolve, reject) => { - resolve([]); - }); + async startTransforms( + reqBody: StartTransformsRequestSchema + ): Promise { + return Promise.resolve({}); + }, + async stopTransforms( + transformsInfo: StopTransformsRequestSchema + ): Promise { + return Promise.resolve({}); }, - getIndices() { - return new Promise((resolve, reject) => { - resolve([]); + async getTransformAuditMessages( + transformId: TransformId + ): Promise { + return Promise.resolve([]); + }, + async esSearch(payload: any): Promise { + return Promise.resolve({ + hits: { + hits: [], + total: { + value: 0, + relation: 'the-relation', + }, + max_score: 0, + }, + timed_out: false, + took: 10, + _shards: { total: 1, successful: 1, failed: 0, skipped: 0 }, }); }, + + async getEsIndices(): Promise { + return Promise.resolve([]); + }, + async getHistogramsForFields( + indexPatternTitle: string, + fields: FieldHistogramRequestConfig[], + query: string | SavedSearchQuery, + samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE + ): Promise { + return Promise.resolve([]); + }, }); export const useApi = () => { diff --git a/x-pack/plugins/transform/public/app/hooks/use_api.ts b/x-pack/plugins/transform/public/app/hooks/use_api.ts index 1d2752b9e939d..4cff5dd9b648e 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_api.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_api.ts @@ -6,20 +6,43 @@ import { useMemo } from 'react'; +import { HttpFetchError } from 'kibana/public'; + import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; -import { - TransformId, - TransformEndpointRequest, - TransformEndpointResult, - DeleteTransformEndpointResult, -} from '../../../common'; +import type { GetTransformsAuditMessagesResponseSchema } from '../../../common/api_schemas/audit_messages'; +import type { + DeleteTransformsRequestSchema, + DeleteTransformsResponseSchema, +} from '../../../common/api_schemas/delete_transforms'; +import type { FieldHistogramsResponseSchema } from '../../../common/api_schemas/field_histograms'; +import type { + StartTransformsRequestSchema, + StartTransformsResponseSchema, +} from '../../../common/api_schemas/start_transforms'; +import type { + StopTransformsRequestSchema, + StopTransformsResponseSchema, +} from '../../../common/api_schemas/stop_transforms'; +import type { + GetTransformsResponseSchema, + PostTransformsPreviewRequestSchema, + PostTransformsPreviewResponseSchema, + PutTransformsRequestSchema, + PutTransformsResponseSchema, +} from '../../../common/api_schemas/transforms'; +import type { + PostTransformsUpdateRequestSchema, + PostTransformsUpdateResponseSchema, +} from '../../../common/api_schemas/update_transforms'; +import type { GetTransformsStatsResponseSchema } from '../../../common/api_schemas/transforms_stats'; +import { TransformId } from '../../../common/types/transform'; import { API_BASE_PATH } from '../../../common/constants'; +import { EsIndex } from '../../../common/types/es_index'; +import type { SearchResponse7 } from '../../../common/shared_imports'; import { useAppDependencies } from '../app_dependencies'; -import { GetTransformsResponse, PreviewRequestBody } from '../common'; -import { EsIndex } from './use_api_types'; import { SavedSearchQuery } from './use_search_items'; // Default sampler shard size used for field histograms @@ -35,81 +58,146 @@ export const useApi = () => { return useMemo( () => ({ - getTransforms(transformId?: TransformId): Promise { - const transformIdString = transformId !== undefined ? `/${transformId}` : ''; - return http.get(`${API_BASE_PATH}transforms${transformIdString}`); + async getTransform( + transformId: TransformId + ): Promise { + try { + return await http.get(`${API_BASE_PATH}transforms/${transformId}`); + } catch (e) { + return e; + } }, - getTransformsStats(transformId?: TransformId): Promise { - if (transformId !== undefined) { - return http.get(`${API_BASE_PATH}transforms/${transformId}/_stats`); + async getTransforms(): Promise { + try { + return await http.get(`${API_BASE_PATH}transforms`); + } catch (e) { + return e; } - - return http.get(`${API_BASE_PATH}transforms/_stats`); }, - createTransform(transformId: TransformId, transformConfig: any): Promise { - return http.put(`${API_BASE_PATH}transforms/${transformId}`, { - body: JSON.stringify(transformConfig), - }); + async getTransformStats( + transformId: TransformId + ): Promise { + try { + return await http.get(`${API_BASE_PATH}transforms/${transformId}/_stats`); + } catch (e) { + return e; + } }, - updateTransform(transformId: TransformId, transformConfig: any): Promise { - return http.post(`${API_BASE_PATH}transforms/${transformId}/_update`, { - body: JSON.stringify(transformConfig), - }); + async getTransformsStats(): Promise { + try { + return await http.get(`${API_BASE_PATH}transforms/_stats`); + } catch (e) { + return e; + } }, - deleteTransforms( - transformsInfo: TransformEndpointRequest[], - deleteDestIndex: boolean | undefined, - deleteDestIndexPattern: boolean | undefined, - forceDelete: boolean - ): Promise { - return http.post(`${API_BASE_PATH}delete_transforms`, { - body: JSON.stringify({ - transformsInfo, - deleteDestIndex, - deleteDestIndexPattern, - forceDelete, - }), - }); + async createTransform( + transformId: TransformId, + transformConfig: PutTransformsRequestSchema + ): Promise { + try { + return await http.put(`${API_BASE_PATH}transforms/${transformId}`, { + body: JSON.stringify(transformConfig), + }); + } catch (e) { + return e; + } }, - getTransformsPreview(obj: PreviewRequestBody): Promise { - return http.post(`${API_BASE_PATH}transforms/_preview`, { - body: JSON.stringify(obj), - }); + async updateTransform( + transformId: TransformId, + transformConfig: PostTransformsUpdateRequestSchema + ): Promise { + try { + return await http.post(`${API_BASE_PATH}transforms/${transformId}/_update`, { + body: JSON.stringify(transformConfig), + }); + } catch (e) { + return e; + } }, - startTransforms( - transformsInfo: TransformEndpointRequest[] - ): Promise { - return http.post(`${API_BASE_PATH}start_transforms`, { - body: JSON.stringify(transformsInfo), - }); + async deleteTransforms( + reqBody: DeleteTransformsRequestSchema + ): Promise { + try { + return await http.post(`${API_BASE_PATH}delete_transforms`, { + body: JSON.stringify(reqBody), + }); + } catch (e) { + return e; + } }, - stopTransforms(transformsInfo: TransformEndpointRequest[]): Promise { - return http.post(`${API_BASE_PATH}stop_transforms`, { - body: JSON.stringify(transformsInfo), - }); + async getTransformsPreview( + obj: PostTransformsPreviewRequestSchema + ): Promise { + try { + return await http.post(`${API_BASE_PATH}transforms/_preview`, { + body: JSON.stringify(obj), + }); + } catch (e) { + return e; + } }, - getTransformAuditMessages(transformId: TransformId): Promise { - return http.get(`${API_BASE_PATH}transforms/${transformId}/messages`); + async startTransforms( + reqBody: StartTransformsRequestSchema + ): Promise { + try { + return await http.post(`${API_BASE_PATH}start_transforms`, { + body: JSON.stringify(reqBody), + }); + } catch (e) { + return e; + } + }, + async stopTransforms( + transformsInfo: StopTransformsRequestSchema + ): Promise { + try { + return await http.post(`${API_BASE_PATH}stop_transforms`, { + body: JSON.stringify(transformsInfo), + }); + } catch (e) { + return e; + } + }, + async getTransformAuditMessages( + transformId: TransformId + ): Promise { + try { + return await http.get(`${API_BASE_PATH}transforms/${transformId}/messages`); + } catch (e) { + return e; + } }, - esSearch(payload: any): Promise { - return http.post(`${API_BASE_PATH}es_search`, { body: JSON.stringify(payload) }); + async esSearch(payload: any): Promise { + try { + return await http.post(`${API_BASE_PATH}es_search`, { body: JSON.stringify(payload) }); + } catch (e) { + return e; + } }, - getIndices(): Promise { - return http.get(`/api/index_management/indices`); + async getEsIndices(): Promise { + try { + return await http.get(`/api/index_management/indices`); + } catch (e) { + return e; + } }, - getHistogramsForFields( + async getHistogramsForFields( indexPatternTitle: string, fields: FieldHistogramRequestConfig[], query: string | SavedSearchQuery, samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE - ) { - return http.post(`${API_BASE_PATH}field_histograms/${indexPatternTitle}`, { - body: JSON.stringify({ - query, - fields, - samplerShardSize, - }), - }); + ): Promise { + try { + return await http.post(`${API_BASE_PATH}field_histograms/${indexPatternTitle}`, { + body: JSON.stringify({ + query, + fields, + samplerShardSize, + }), + }); + } catch (e) { + return e; + } }, }), [http] diff --git a/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx index fdf77c8ebee51..1a97ba7806fef 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx @@ -7,11 +7,11 @@ import React, { useCallback, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public'; -import { - DeleteTransformEndpointResult, +import type { DeleteTransformStatus, - TransformEndpointRequest, -} from '../../../common'; + DeleteTransformsRequestSchema, +} from '../../../common/api_schemas/delete_transforms'; +import { isDeleteTransformsResponseSchema } from '../../../common/api_schemas/type_guards'; import { extractErrorMessage } from '../../shared_imports'; import { getErrorMessage } from '../../../common/utils/errors'; import { useAppDependencies, useToastNotifications } from '../app_dependencies'; @@ -109,173 +109,157 @@ export const useDeleteTransforms = () => { const toastNotifications = useToastNotifications(); const api = useApi(); - return async ( - transforms: TransformListRow[], - shouldDeleteDestIndex: boolean, - shouldDeleteDestIndexPattern: boolean, - shouldForceDelete = false - ) => { - const transformsInfo: TransformEndpointRequest[] = transforms.map((tf) => ({ - id: tf.config.id, - state: tf.stats.state, - })); + return async (reqBody: DeleteTransformsRequestSchema) => { + const results = await api.deleteTransforms(reqBody); - try { - const results: DeleteTransformEndpointResult = await api.deleteTransforms( - transformsInfo, - shouldDeleteDestIndex, - shouldDeleteDestIndexPattern, - shouldForceDelete - ); - const isBulk = Object.keys(results).length > 1; - const successCount: Record = { - transformDeleted: 0, - destIndexDeleted: 0, - destIndexPatternDeleted: 0, - }; - for (const transformId in results) { - // hasOwnProperty check to ensure only properties on object itself, and not its prototypes - if (results.hasOwnProperty(transformId)) { - const status = results[transformId]; - const destinationIndex = status.destinationIndex; + if (!isDeleteTransformsResponseSchema(results)) { + toastNotifications.addDanger({ + title: i18n.translate('xpack.transform.transformList.deleteTransformGenericErrorMessage', { + defaultMessage: 'An error occurred calling the API endpoint to delete transforms.', + }), + text: toMountPoint( + + ), + }); + return; + } - // if we are only deleting one transform, show the success toast messages - if (!isBulk && status.transformDeleted) { - if (status.transformDeleted?.success) { - toastNotifications.addSuccess( - i18n.translate('xpack.transform.transformList.deleteTransformSuccessMessage', { - defaultMessage: 'Request to delete transform {transformId} acknowledged.', - values: { transformId }, - }) - ); - } - if (status.destIndexDeleted?.success) { - toastNotifications.addSuccess( - i18n.translate( - 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexSuccessMessage', - { - defaultMessage: - 'Request to delete destination index {destinationIndex} acknowledged.', - values: { destinationIndex }, - } - ) - ); - } - if (status.destIndexPatternDeleted?.success) { - toastNotifications.addSuccess( - i18n.translate( - 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexPatternSuccessMessage', - { - defaultMessage: - 'Request to delete index pattern {destinationIndex} acknowledged.', - values: { destinationIndex }, - } - ) - ); - } - } else { - (Object.keys(successCount) as SuccessCountField[]).forEach((key) => { - if (status[key]?.success) { - successCount[key] = successCount[key] + 1; - } - }); - } - if (status.transformDeleted?.error) { - const error = extractErrorMessage(status.transformDeleted.error); - toastNotifications.addDanger({ - title: i18n.translate('xpack.transform.transformList.deleteTransformErrorMessage', { - defaultMessage: 'An error occurred deleting the transform {transformId}', + const isBulk = Object.keys(results).length > 1; + const successCount: Record = { + transformDeleted: 0, + destIndexDeleted: 0, + destIndexPatternDeleted: 0, + }; + for (const transformId in results) { + // hasOwnProperty check to ensure only properties on object itself, and not its prototypes + if (results.hasOwnProperty(transformId)) { + const status = results[transformId]; + const destinationIndex = status.destinationIndex; + + // if we are only deleting one transform, show the success toast messages + if (!isBulk && status.transformDeleted) { + if (status.transformDeleted?.success) { + toastNotifications.addSuccess( + i18n.translate('xpack.transform.transformList.deleteTransformSuccessMessage', { + defaultMessage: 'Request to delete transform {transformId} acknowledged.', values: { transformId }, - }), - text: toMountPoint( - - ), - }); + }) + ); } - - if (status.destIndexDeleted?.error) { - const error = extractErrorMessage(status.destIndexDeleted.error); - toastNotifications.addDanger({ - title: i18n.translate( - 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexErrorMessage', + if (status.destIndexDeleted?.success) { + toastNotifications.addSuccess( + i18n.translate( + 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexSuccessMessage', { - defaultMessage: 'An error occurred deleting destination index {destinationIndex}', + defaultMessage: + 'Request to delete destination index {destinationIndex} acknowledged.', values: { destinationIndex }, } - ), - text: toMountPoint( - - ), - }); + ) + ); } - - if (status.destIndexPatternDeleted?.error) { - const error = extractErrorMessage(status.destIndexPatternDeleted.error); - toastNotifications.addDanger({ - title: i18n.translate( - 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexPatternErrorMessage', + if (status.destIndexPatternDeleted?.success) { + toastNotifications.addSuccess( + i18n.translate( + 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexPatternSuccessMessage', { - defaultMessage: 'An error occurred deleting index pattern {destinationIndex}', + defaultMessage: + 'Request to delete index pattern {destinationIndex} acknowledged.', values: { destinationIndex }, } - ), - text: toMountPoint( - - ), - }); + ) + ); } + } else { + (Object.keys(successCount) as SuccessCountField[]).forEach((key) => { + if (status[key]?.success) { + successCount[key] = successCount[key] + 1; + } + }); } - } - - // if we are deleting multiple transforms, combine the success messages - if (isBulk) { - if (successCount.transformDeleted > 0) { - toastNotifications.addSuccess( - i18n.translate('xpack.transform.transformList.bulkDeleteTransformSuccessMessage', { - defaultMessage: - 'Successfully deleted {count} {count, plural, one {transform} other {transforms}}.', - values: { count: successCount.transformDeleted }, - }) - ); + if (status.transformDeleted?.error) { + const error = extractErrorMessage(status.transformDeleted.error); + toastNotifications.addDanger({ + title: i18n.translate('xpack.transform.transformList.deleteTransformErrorMessage', { + defaultMessage: 'An error occurred deleting the transform {transformId}', + values: { transformId }, + }), + text: toMountPoint( + + ), + }); } - if (successCount.destIndexDeleted > 0) { - toastNotifications.addSuccess( - i18n.translate('xpack.transform.transformList.bulkDeleteDestIndexSuccessMessage', { - defaultMessage: - 'Successfully deleted {count} destination {count, plural, one {index} other {indices}}.', - values: { count: successCount.destIndexDeleted }, - }) - ); + if (status.destIndexDeleted?.error) { + const error = extractErrorMessage(status.destIndexDeleted.error); + toastNotifications.addDanger({ + title: i18n.translate( + 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexErrorMessage', + { + defaultMessage: 'An error occurred deleting destination index {destinationIndex}', + values: { destinationIndex }, + } + ), + text: toMountPoint( + + ), + }); } - if (successCount.destIndexPatternDeleted > 0) { - toastNotifications.addSuccess( - i18n.translate( - 'xpack.transform.transformList.bulkDeleteDestIndexPatternSuccessMessage', + + if (status.destIndexPatternDeleted?.error) { + const error = extractErrorMessage(status.destIndexPatternDeleted.error); + toastNotifications.addDanger({ + title: i18n.translate( + 'xpack.transform.deleteTransform.deleteAnalyticsWithIndexPatternErrorMessage', { - defaultMessage: - 'Successfully deleted {count} destination index {count, plural, one {pattern} other {patterns}}.', - values: { count: successCount.destIndexPatternDeleted }, + defaultMessage: 'An error occurred deleting index pattern {destinationIndex}', + values: { destinationIndex }, } - ) - ); + ), + text: toMountPoint( + + ), + }); } } + } - refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.REFRESH); - } catch (e) { - toastNotifications.addDanger({ - title: i18n.translate('xpack.transform.transformList.deleteTransformGenericErrorMessage', { - defaultMessage: 'An error occurred calling the API endpoint to delete transforms.', - }), - text: toMountPoint( - - ), - }); + // if we are deleting multiple transforms, combine the success messages + if (isBulk) { + if (successCount.transformDeleted > 0) { + toastNotifications.addSuccess( + i18n.translate('xpack.transform.transformList.bulkDeleteTransformSuccessMessage', { + defaultMessage: + 'Successfully deleted {count} {count, plural, one {transform} other {transforms}}.', + values: { count: successCount.transformDeleted }, + }) + ); + } + + if (successCount.destIndexDeleted > 0) { + toastNotifications.addSuccess( + i18n.translate('xpack.transform.transformList.bulkDeleteDestIndexSuccessMessage', { + defaultMessage: + 'Successfully deleted {count} destination {count, plural, one {index} other {indices}}.', + values: { count: successCount.destIndexDeleted }, + }) + ); + } + if (successCount.destIndexPatternDeleted > 0) { + toastNotifications.addSuccess( + i18n.translate('xpack.transform.transformList.bulkDeleteDestIndexPatternSuccessMessage', { + defaultMessage: + 'Successfully deleted {count} destination index {count, plural, one {pattern} other {patterns}}.', + values: { count: successCount.destIndexPatternDeleted }, + }) + ); + } } + + refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.REFRESH); }; }; diff --git a/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts b/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts index bd19a7f8bf4d8..5f3a9a6abfdb4 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts @@ -4,52 +4,24 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - TransformListRow, - TransformStats, - TRANSFORM_MODE, - isTransformStats, - TransformPivotConfig, - refreshTransformList$, - REFRESH_TRANSFORM_LIST_STATE, -} from '../common'; - -import { useApi } from './use_api'; +import { HttpFetchError } from 'src/core/public'; -interface GetTransformsResponse { - count: number; - transforms: TransformPivotConfig[]; -} - -interface GetTransformsStatsResponseOk { - node_failures?: object; - count: number; - transforms: TransformStats[]; -} - -const isGetTransformsStatsResponseOk = (arg: any): arg is GetTransformsStatsResponseOk => { - return ( - {}.hasOwnProperty.call(arg, 'count') && - {}.hasOwnProperty.call(arg, 'transforms') && - Array.isArray(arg.transforms) - ); -}; +import { + isGetTransformsResponseSchema, + isGetTransformsStatsResponseSchema, +} from '../../../common/api_schemas/type_guards'; +import { TRANSFORM_MODE } from '../../../common/constants'; +import { isTransformStats } from '../../../common/types/transform_stats'; -interface GetTransformsStatsResponseError { - statusCode: number; - error: string; - message: string; -} +import { TransformListRow, refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../common'; -type GetTransformsStatsResponse = GetTransformsStatsResponseOk | GetTransformsStatsResponseError; +import { useApi } from './use_api'; export type GetTransforms = (forceRefresh?: boolean) => void; export const useGetTransforms = ( setTransforms: React.Dispatch>, - setErrorMessage: React.Dispatch< - React.SetStateAction - >, + setErrorMessage: React.Dispatch>, setIsInitialized: React.Dispatch>, blockRefresh: boolean ): GetTransforms => { @@ -66,45 +38,57 @@ export const useGetTransforms = ( return; } - try { - const transformConfigs: GetTransformsResponse = await api.getTransforms(); - const transformStats: GetTransformsStatsResponse = await api.getTransformsStats(); - - const tableRows = transformConfigs.transforms.reduce((reducedtableRows, config) => { - const stats = isGetTransformsStatsResponseOk(transformStats) - ? transformStats.transforms.find((d) => config.id === d.id) - : undefined; - - // A newly created transform might not have corresponding stats yet. - // If that's the case we just skip the transform and don't add it to the transform list yet. - if (!isTransformStats(stats)) { - return reducedtableRows; - } - - // Table with expandable rows requires `id` on the outer most level - reducedtableRows.push({ - id: config.id, - config, - mode: - typeof config.sync !== 'undefined' ? TRANSFORM_MODE.CONTINUOUS : TRANSFORM_MODE.BATCH, - stats, - }); - return reducedtableRows; - }, [] as TransformListRow[]); + const transformConfigs = await api.getTransforms(); + const transformStats = await api.getTransformsStats(); - setTransforms(tableRows); - setErrorMessage(undefined); - setIsInitialized(true); - refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.IDLE); - } catch (e) { + if ( + !isGetTransformsResponseSchema(transformConfigs) || + !isGetTransformsStatsResponseSchema(transformStats) + ) { // An error is followed immediately by setting the state to idle. // This way we're able to treat ERROR as a one-time-event like REFRESH. refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.ERROR); refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.IDLE); setTransforms([]); - setErrorMessage(e); + setIsInitialized(true); + + if (!isGetTransformsResponseSchema(transformConfigs)) { + setErrorMessage(transformConfigs); + } else if (!isGetTransformsStatsResponseSchema(transformStats)) { + setErrorMessage(transformStats); + } + + return; } + + const tableRows = transformConfigs.transforms.reduce((reducedtableRows, config) => { + const stats = isGetTransformsStatsResponseSchema(transformStats) + ? transformStats.transforms.find((d) => config.id === d.id) + : undefined; + + // A newly created transform might not have corresponding stats yet. + // If that's the case we just skip the transform and don't add it to the transform list yet. + if (!isTransformStats(stats)) { + return reducedtableRows; + } + + // Table with expandable rows requires `id` on the outer most level + reducedtableRows.push({ + id: config.id, + config, + mode: + typeof config.sync !== 'undefined' ? TRANSFORM_MODE.CONTINUOUS : TRANSFORM_MODE.BATCH, + stats, + }); + return reducedtableRows; + }, [] as TransformListRow[]); + + setTransforms(tableRows); + setErrorMessage(undefined); + setIsInitialized(true); + refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.IDLE); + concurrentLoads--; if (concurrentLoads > 0) { diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index 946f7991d049d..ce233d0cf7caa 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -8,6 +8,11 @@ import { useEffect } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; +import { + isEsSearchResponse, + isFieldHistogramsResponseSchema, +} from '../../../common/api_schemas/type_guards'; + import { getFieldType, getDataGridSchemaFromKibanaFieldType, @@ -16,7 +21,6 @@ import { useDataGrid, useRenderCellValue, EsSorting, - SearchResponse7, UseIndexDataReturnType, INDEX_STATUS, } from '../../shared_imports'; @@ -29,8 +33,6 @@ import { useApi } from './use_api'; import { useToastNotifications } from '../app_dependencies'; -type IndexSearchResponse = SearchResponse7; - export const useIndexData = ( indexPattern: SearchItems['indexPattern'], query: PivotQuery @@ -90,37 +92,39 @@ export const useIndexData = ( }, }; - try { - const resp: IndexSearchResponse = await api.esSearch(esSearchRequest); - - const docs = resp.hits.hits.map((d) => d._source); + const resp = await api.esSearch(esSearchRequest); - setRowCount(resp.hits.total.value); - setTableItems(docs); - setStatus(INDEX_STATUS.LOADED); - } catch (e) { - setErrorMessage(getErrorMessage(e)); + if (!isEsSearchResponse(resp)) { + setErrorMessage(getErrorMessage(resp)); setStatus(INDEX_STATUS.ERROR); return; } + + const docs = resp.hits.hits.map((d) => d._source); + + setRowCount(resp.hits.total.value); + setTableItems(docs); + setStatus(INDEX_STATUS.LOADED); }; const fetchColumnChartsData = async function () { - try { - const columnChartsData = await api.getHistogramsForFields( - indexPattern.title, - columns - .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) - .map((cT) => ({ - fieldName: cT.id, - type: getFieldType(cT.schema), - })), - isDefaultQuery(query) ? matchAllQuery : query - ); - setColumnCharts(columnChartsData); - } catch (e) { - showDataGridColumnChartErrorMessageToast(e, toastNotifications); + const columnChartsData = await api.getHistogramsForFields( + indexPattern.title, + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })), + isDefaultQuery(query) ? matchAllQuery : query + ); + + if (!isFieldHistogramsResponseSchema(columnChartsData)) { + showDataGridColumnChartErrorMessageToast(columnChartsData, toastNotifications); + return; } + + setColumnCharts(columnChartsData); }; useEffect(() => { diff --git a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts index a0e7c5dde494a..c51bf7d7e6741 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts @@ -13,11 +13,13 @@ import { i18n } from '@kbn/i18n'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; +import type { PreviewMappingsProperties } from '../../../common/api_schemas/transforms'; +import { isPostTransformsPreviewResponseSchema } from '../../../common/api_schemas/type_guards'; import { dictionaryToArray } from '../../../common/types/common'; -import { formatHumanReadableDateTimeSeconds } from '../../shared_imports'; import { getNestedProperty } from '../../../common/utils/object_utils'; import { + formatHumanReadableDateTimeSeconds, multiColumnSortFactory, useDataGrid, RenderCellValue, @@ -27,12 +29,11 @@ import { import { getErrorMessage } from '../../../common/utils/errors'; import { - getPreviewRequestBody, + getPreviewTransformRequestBody, PivotAggsConfigDict, PivotGroupByConfigDict, PivotGroupByConfig, PivotQuery, - PreviewMappings, PivotAggsConfig, } from '../common'; @@ -74,21 +75,23 @@ export const usePivotData = ( aggs: PivotAggsConfigDict, groupBy: PivotGroupByConfigDict ): UseIndexDataReturnType => { - const [previewMappings, setPreviewMappings] = useState({ properties: {} }); + const [previewMappingsProperties, setPreviewMappingsProperties] = useState< + PreviewMappingsProperties + >({}); const api = useApi(); const aggsArr = useMemo(() => dictionaryToArray(aggs), [aggs]); const groupByArr = useMemo(() => dictionaryToArray(groupBy), [groupBy]); // Filters mapping properties of type `object`, which get returned for nested field parents. - const columnKeys = Object.keys(previewMappings.properties).filter( - (key) => previewMappings.properties[key].type !== 'object' + const columnKeys = Object.keys(previewMappingsProperties).filter( + (key) => previewMappingsProperties[key].type !== 'object' ); columnKeys.sort(sortColumns(groupByArr)); // EuiDataGrid State const columns: EuiDataGridColumn[] = columnKeys.map((id) => { - const field = previewMappings.properties[id]; + const field = previewMappingsProperties[id]; // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json'] // To fall back to the default string schema it needs to be undefined. @@ -159,28 +162,35 @@ export const usePivotData = ( setNoDataMessage(''); setStatus(INDEX_STATUS.LOADING); - try { - const previewRequest = getPreviewRequestBody(indexPatternTitle, query, groupByArr, aggsArr); - const resp = await api.getTransformsPreview(previewRequest); - setTableItems(resp.preview); - setRowCount(resp.preview.length); - setPreviewMappings(resp.generated_dest_index.mappings); - setStatus(INDEX_STATUS.LOADED); - - if (resp.preview.length === 0) { - setNoDataMessage( - i18n.translate('xpack.transform.pivotPreview.PivotPreviewNoDataCalloutBody', { - defaultMessage: - 'The preview request did not return any data. Please ensure the optional query returns data and that values exist for the field used by group-by and aggregation fields.', - }) - ); - } - } catch (e) { - setErrorMessage(getErrorMessage(e)); + const previewRequest = getPreviewTransformRequestBody( + indexPatternTitle, + query, + groupByArr, + aggsArr + ); + const resp = await api.getTransformsPreview(previewRequest); + + if (!isPostTransformsPreviewResponseSchema(resp)) { + setErrorMessage(getErrorMessage(resp)); setTableItems([]); setRowCount(0); - setPreviewMappings({ properties: {} }); + setPreviewMappingsProperties({}); setStatus(INDEX_STATUS.ERROR); + return; + } + + setTableItems(resp.preview); + setRowCount(resp.preview.length); + setPreviewMappingsProperties(resp.generated_dest_index.mappings.properties); + setStatus(INDEX_STATUS.LOADED); + + if (resp.preview.length === 0) { + setNoDataMessage( + i18n.translate('xpack.transform.pivotPreview.PivotPreviewNoDataCalloutBody', { + defaultMessage: + 'The preview request did not return any data. Please ensure the optional query returns data and that values exist for the field used by group-by and aggregation fields.', + }) + ); } }; @@ -236,19 +246,19 @@ export const usePivotData = ( if ( [ES_FIELD_TYPES.DATE, ES_FIELD_TYPES.DATE_NANOS].includes( - previewMappings.properties[columnId].type + previewMappingsProperties[columnId].type ) ) { return formatHumanReadableDateTimeSeconds(moment(cellValue).unix() * 1000); } - if (previewMappings.properties[columnId].type === ES_FIELD_TYPES.BOOLEAN) { + if (previewMappingsProperties[columnId].type === ES_FIELD_TYPES.BOOLEAN) { return cellValue ? 'true' : 'false'; } return cellValue; }; - }, [pageData, pagination.pageIndex, pagination.pageSize, previewMappings.properties]); + }, [pageData, pagination.pageIndex, pagination.pageSize, previewMappingsProperties]); return { ...dataGrid, diff --git a/x-pack/plugins/transform/public/app/hooks/use_start_transform.ts b/x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx similarity index 52% rename from x-pack/plugins/transform/public/app/hooks/use_start_transform.ts rename to x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx index a0ffe1fdfa336..71ed220b6b4df 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_start_transform.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx @@ -4,25 +4,45 @@ * you may not use this file except in compliance with the Elastic License. */ +import React from 'react'; + import { i18n } from '@kbn/i18n'; -import { TransformEndpointRequest, TransformEndpointResult } from '../../../common'; +import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public'; + +import type { StartTransformsRequestSchema } from '../../../common/api_schemas/start_transforms'; +import { isStartTransformsResponseSchema } from '../../../common/api_schemas/type_guards'; + +import { getErrorMessage } from '../../../common/utils/errors'; -import { useToastNotifications } from '../app_dependencies'; -import { TransformListRow, refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../common'; +import { useAppDependencies, useToastNotifications } from '../app_dependencies'; +import { refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../common'; +import { ToastNotificationText } from '../components'; import { useApi } from './use_api'; export const useStartTransforms = () => { + const deps = useAppDependencies(); const toastNotifications = useToastNotifications(); const api = useApi(); - return async (transforms: TransformListRow[]) => { - const transformsInfo: TransformEndpointRequest[] = transforms.map((tf) => ({ - id: tf.config.id, - state: tf.stats.state, - })); - const results: TransformEndpointResult = await api.startTransforms(transformsInfo); + return async (transformsInfo: StartTransformsRequestSchema) => { + const results = await api.startTransforms(transformsInfo); + + if (!isStartTransformsResponseSchema(results)) { + toastNotifications.addDanger({ + title: i18n.translate( + 'xpack.transform.stepCreateForm.startTransformResponseSchemaErrorMessage', + { + defaultMessage: 'An error occurred calling the start transforms request.', + } + ), + text: toMountPoint( + + ), + }); + return; + } for (const transformId in results) { // hasOwnProperty check to ensure only properties on object itself, and not its prototypes diff --git a/x-pack/plugins/transform/public/app/hooks/use_stop_transform.ts b/x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx similarity index 53% rename from x-pack/plugins/transform/public/app/hooks/use_stop_transform.ts rename to x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx index 0df9834647704..be223c5eddfdd 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_stop_transform.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx @@ -4,25 +4,45 @@ * you may not use this file except in compliance with the Elastic License. */ +import React from 'react'; + import { i18n } from '@kbn/i18n'; -import { TransformEndpointRequest, TransformEndpointResult } from '../../../common'; +import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public'; + +import type { StopTransformsRequestSchema } from '../../../common/api_schemas/stop_transforms'; +import { isStopTransformsResponseSchema } from '../../../common/api_schemas/type_guards'; + +import { getErrorMessage } from '../../../common/utils/errors'; -import { useToastNotifications } from '../app_dependencies'; -import { TransformListRow, refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../common'; +import { useAppDependencies, useToastNotifications } from '../app_dependencies'; +import { refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../common'; +import { ToastNotificationText } from '../components'; import { useApi } from './use_api'; export const useStopTransforms = () => { + const deps = useAppDependencies(); const toastNotifications = useToastNotifications(); const api = useApi(); - return async (transforms: TransformListRow[]) => { - const transformsInfo: TransformEndpointRequest[] = transforms.map((df) => ({ - id: df.config.id, - state: df.stats.state, - })); - const results: TransformEndpointResult = await api.stopTransforms(transformsInfo); + return async (transformsInfo: StopTransformsRequestSchema) => { + const results = await api.stopTransforms(transformsInfo); + + if (!isStopTransformsResponseSchema(results)) { + toastNotifications.addDanger({ + title: i18n.translate( + 'xpack.transform.transformList.stopTransformResponseSchemaErrorMessage', + { + defaultMessage: 'An error occurred called the stop transforms request.', + } + ), + text: toMountPoint( + + ), + }); + return; + } for (const transformId in results) { // hasOwnProperty check to ensure only properties on object itself, and not its prototypes diff --git a/x-pack/plugins/transform/public/app/lib/authorization/components/authorization_provider.tsx b/x-pack/plugins/transform/public/app/lib/authorization/components/authorization_provider.tsx index 6553d4474d392..790fcaf5fa83c 100644 --- a/x-pack/plugins/transform/public/app/lib/authorization/components/authorization_provider.tsx +++ b/x-pack/plugins/transform/public/app/lib/authorization/components/authorization_provider.tsx @@ -6,7 +6,7 @@ import React, { createContext } from 'react'; -import { Privileges } from '../../../../../common'; +import { Privileges } from '../../../../../common/types/privileges'; import { useRequest } from '../../../hooks'; diff --git a/x-pack/plugins/transform/public/app/lib/authorization/components/common.ts b/x-pack/plugins/transform/public/app/lib/authorization/components/common.ts index 282a737d0bf1e..841c6ed01766a 100644 --- a/x-pack/plugins/transform/public/app/lib/authorization/components/common.ts +++ b/x-pack/plugins/transform/public/app/lib/authorization/components/common.ts @@ -6,7 +6,7 @@ import { i18n } from '@kbn/i18n'; -import { Privileges } from '../../../../../common'; +import { Privileges } from '../../../../../common/types/privileges'; export interface Capabilities { canGetTransform: boolean; diff --git a/x-pack/plugins/transform/public/app/lib/authorization/components/with_privileges.tsx b/x-pack/plugins/transform/public/app/lib/authorization/components/with_privileges.tsx index 89c6ac3a054f7..beeacc76bdc95 100644 --- a/x-pack/plugins/transform/public/app/lib/authorization/components/with_privileges.tsx +++ b/x-pack/plugins/transform/public/app/lib/authorization/components/with_privileges.tsx @@ -10,7 +10,7 @@ import { EuiPageContent } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { MissingPrivileges } from '../../../../../common'; +import { MissingPrivileges } from '../../../../../common/types/privileges'; import { SectionLoading } from '../../../components'; diff --git a/x-pack/plugins/transform/public/app/sections/clone_transform/clone_transform_section.tsx b/x-pack/plugins/transform/public/app/sections/clone_transform/clone_transform_section.tsx index 19ba31d36e6e9..9a97c66bfb10b 100644 --- a/x-pack/plugins/transform/public/app/sections/clone_transform/clone_transform_section.tsx +++ b/x-pack/plugins/transform/public/app/sections/clone_transform/clone_transform_section.tsx @@ -21,40 +21,20 @@ import { EuiTitle, } from '@elastic/eui'; +import { APP_CREATE_TRANSFORM_CLUSTER_PRIVILEGES } from '../../../../common/constants'; +import { TransformPivotConfig } from '../../../../common/types/transform'; + +import { isHttpFetchError } from '../../common/request'; import { useApi } from '../../hooks/use_api'; import { useDocumentationLinks } from '../../hooks/use_documentation_links'; import { useSearchItems } from '../../hooks/use_search_items'; -import { APP_CREATE_TRANSFORM_CLUSTER_PRIVILEGES } from '../../../../common/constants'; - import { useAppDependencies } from '../../app_dependencies'; -import { TransformPivotConfig } from '../../common'; import { breadcrumbService, docTitleService, BREADCRUMB_SECTION } from '../../services/navigation'; import { PrivilegesWrapper } from '../../lib/authorization'; import { Wizard } from '../create_transform/components/wizard'; -interface GetTransformsResponseOk { - count: number; - transforms: TransformPivotConfig[]; -} - -interface GetTransformsResponseError { - error: { - msg: string; - path: string; - query: any; - statusCode: number; - response: string; - }; -} - -function isGetTransformsResponseError(arg: any): arg is GetTransformsResponseError { - return arg.error !== undefined; -} - -type GetTransformsResponse = GetTransformsResponseOk | GetTransformsResponseError; - type Props = RouteComponentProps<{ transformId: string }>; export const CloneTransformSection: FC = ({ match }) => { // Set breadcrumb and page title @@ -84,15 +64,15 @@ export const CloneTransformSection: FC = ({ match }) => { } = useSearchItems(undefined); const fetchTransformConfig = async () => { - try { - const transformConfigs: GetTransformsResponse = await api.getTransforms(transformId); - if (isGetTransformsResponseError(transformConfigs)) { - setTransformConfig(undefined); - setErrorMessage(transformConfigs.error.msg); - setIsInitialized(true); - return; - } + const transformConfigs = await api.getTransform(transformId); + if (isHttpFetchError(transformConfigs)) { + setTransformConfig(undefined); + setErrorMessage(transformConfigs.message); + setIsInitialized(true); + return; + } + try { await loadIndexPatterns(savedObjectsClient, indexPatterns); const indexPatternTitle = Array.isArray(transformConfigs.transforms[0].source.index) ? transformConfigs.transforms[0].source.index.join(',') diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.test.tsx index fa0fe7bdf6126..49d59706befb8 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.test.tsx @@ -7,7 +7,10 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { AggName, PivotAggsConfig, PIVOT_SUPPORTED_AGGS } from '../../../../common'; +import { AggName } from '../../../../../../common/types/aggregations'; +import { PIVOT_SUPPORTED_AGGS } from '../../../../../../common/types/pivot_aggs'; + +import { PivotAggsConfig } from '../../../../common'; import { AggLabelForm } from './agg_label_form'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.tsx index e50ba9e137331..4e5e3f71cd6e2 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/agg_label_form.tsx @@ -10,8 +10,9 @@ import { i18n } from '@kbn/i18n'; import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiPopover, EuiTextColor } from '@elastic/eui'; +import { AggName } from '../../../../../../common/types/aggregations'; + import { - AggName, isPivotAggsConfigWithUiSupport, PivotAggsConfig, PivotAggsConfigWithUiSupportDict, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.test.tsx index 32c7ca5972e00..93de3d4fcfc9f 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.test.tsx @@ -7,7 +7,9 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { PivotAggsConfig, PIVOT_SUPPORTED_AGGS } from '../../../../common'; +import { PIVOT_SUPPORTED_AGGS } from '../../../../../../common/types/pivot_aggs'; + +import { PivotAggsConfig } from '../../../../common'; import { AggListForm, AggListProps } from './list_form'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.tsx index a02f4455250d7..f6ae1f292b0e6 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_form.tsx @@ -8,8 +8,9 @@ import React, { Fragment } from 'react'; import { EuiPanel, EuiSpacer } from '@elastic/eui'; +import { AggName } from '../../../../../../common/types/aggregations'; + import { - AggName, PivotAggsConfig, PivotAggsConfigDict, PivotAggsConfigWithUiSupportDict, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.test.tsx index 923d52ba5cec1..8c644c358e658 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.test.tsx @@ -7,7 +7,9 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { PivotAggsConfig, PIVOT_SUPPORTED_AGGS } from '../../../../common'; +import { PIVOT_SUPPORTED_AGGS } from '../../../../../../common/types/pivot_aggs'; + +import { PivotAggsConfig } from '../../../../common'; import { AggListSummary, AggListSummaryProps } from './list_summary'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.tsx index 7d07d79e7d283..fb6e141a54b04 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/list_summary.tsx @@ -8,7 +8,9 @@ import React, { Fragment } from 'react'; import { EuiForm, EuiPanel, EuiSpacer } from '@elastic/eui'; -import { AggName, PivotAggsConfigDict } from '../../../../common'; +import { AggName } from '../../../../../../common/types/aggregations'; + +import { PivotAggsConfigDict } from '../../../../common'; export interface AggListSummaryProps { list: PivotAggsConfigDict; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.test.tsx index b3e770a269681..8f2fbfb7084e6 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.test.tsx @@ -7,7 +7,10 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { AggName, PIVOT_SUPPORTED_AGGS, PivotAggsConfig } from '../../../../common'; +import { AggName } from '../../../../../../common/types/aggregations'; +import { PIVOT_SUPPORTED_AGGS } from '../../../../../../common/types/pivot_aggs'; + +import { PivotAggsConfig } from '../../../../common'; import { PopoverForm } from './popover_form'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx index 50064274cf98e..30e8c2b594db7 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx @@ -20,10 +20,14 @@ import { import { cloneDeep } from 'lodash'; import { useUpdateEffect } from 'react-use'; +import { AggName } from '../../../../../../common/types/aggregations'; import { dictionaryToArray } from '../../../../../../common/types/common'; +import { + PivotSupportedAggs, + PIVOT_SUPPORTED_AGGS, +} from '../../../../../../common/types/pivot_aggs'; import { - AggName, isAggName, isPivotAggsConfigPercentiles, isPivotAggsConfigWithUiSupport, @@ -31,9 +35,8 @@ import { PERCENTILES_AGG_DEFAULT_PERCENTS, PivotAggsConfig, PivotAggsConfigWithUiSupportDict, - PIVOT_SUPPORTED_AGGS, } from '../../../../common'; -import { isPivotAggsWithExtendedForm, PivotSupportedAggs } from '../../../../common/pivot_aggs'; +import { isPivotAggsWithExtendedForm } from '../../../../common/pivot_aggs'; import { getAggFormConfig } from '../step_define/common/get_agg_form_config'; interface Props { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/group_by_label_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/group_by_label_form.tsx index c79da06ac8080..ff66ed6779e14 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/group_by_label_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/group_by_label_form.tsx @@ -10,8 +10,9 @@ import { i18n } from '@kbn/i18n'; import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiPopover, EuiTextColor } from '@elastic/eui'; +import { AggName } from '../../../../../../common/types/aggregations'; + import { - AggName, isGroupByDateHistogram, isGroupByHistogram, PivotGroupByConfig, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/list_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/list_form.tsx index 2dc1a4332f6ad..a60989c76ab13 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/list_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/list_form.tsx @@ -8,8 +8,9 @@ import React, { Fragment } from 'react'; import { EuiPanel, EuiSpacer } from '@elastic/eui'; +import { AggName } from '../../../../../../common/types/aggregations'; + import { - AggName, PivotGroupByConfig, PivotGroupByConfigDict, PivotGroupByConfigWithUiSupportDict, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.test.tsx index 090f3b19f47fb..13829222f11f5 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.test.tsx @@ -7,7 +7,9 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { AggName, PIVOT_SUPPORTED_GROUP_BY_AGGS, PivotGroupByConfig } from '../../../../common'; +import { AggName } from '../../../../../../common/types/aggregations'; + +import { PIVOT_SUPPORTED_GROUP_BY_AGGS, PivotGroupByConfig } from '../../../../common'; import { isIntervalValid, PopoverForm } from './popover_form'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.tsx index 0452638e90dfb..f0a96fa6ab875 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/group_by_list/popover_form.tsx @@ -18,10 +18,10 @@ import { EuiSpacer, } from '@elastic/eui'; +import { AggName } from '../../../../../../common/types/aggregations'; import { dictionaryToArray } from '../../../../../../common/types/common'; import { - AggName, dateHistogramIntervalFormatRegex, getEsAggFromGroupByConfig, isGroupByDateHistogram, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx index 2fa1b7c713370..675bd0f9f88ed 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx @@ -30,6 +30,12 @@ import { import { toMountPoint } from '../../../../../../../../../src/plugins/kibana_react/public'; +import type { PutTransformsResponseSchema } from '../../../../../../common/api_schemas/transforms'; +import { + isGetTransformsStatsResponseSchema, + isPutTransformsResponseSchema, + isStartTransformsResponseSchema, +} from '../../../../../../common/api_schemas/type_guards'; import { PROGRESS_REFRESH_INTERVAL_MS } from '../../../../../../common/constants'; import { getErrorMessage } from '../../../../../../common/utils/errors'; @@ -93,34 +99,28 @@ export const StepCreateForm: FC = React.memo( async function createTransform() { setLoading(true); - try { - const resp = await api.createTransform(transformId, transformConfig); - if (resp.errors !== undefined && Array.isArray(resp.errors)) { - if (resp.errors.length === 1) { - throw resp.errors[0]; - } + const resp = await api.createTransform(transformId, transformConfig); - if (resp.errors.length > 1) { - throw resp.errors; - } + if (!isPutTransformsResponseSchema(resp) || resp.errors.length > 0) { + let respErrors: + | PutTransformsResponseSchema['errors'] + | PutTransformsResponseSchema['errors'][number] + | undefined; + + if (isPutTransformsResponseSchema(resp) && resp.errors.length > 0) { + respErrors = resp.errors.length === 1 ? resp.errors[0] : resp.errors; } - toastNotifications.addSuccess( - i18n.translate('xpack.transform.stepCreateForm.createTransformSuccessMessage', { - defaultMessage: 'Request to create transform {transformId} acknowledged.', - values: { transformId }, - }) - ); - setCreated(true); - setLoading(false); - } catch (e) { toastNotifications.addDanger({ title: i18n.translate('xpack.transform.stepCreateForm.createTransformErrorMessage', { defaultMessage: 'An error occurred creating the transform {transformId}:', values: { transformId }, }), text: toMountPoint( - + ), }); setCreated(false); @@ -128,6 +128,15 @@ export const StepCreateForm: FC = React.memo( return false; } + toastNotifications.addSuccess( + i18n.translate('xpack.transform.stepCreateForm.createTransformSuccessMessage', { + defaultMessage: 'Request to create transform {transformId} acknowledged.', + values: { transformId }, + }) + ); + setCreated(true); + setLoading(false); + if (createIndexPattern) { createKibanaIndexPattern(); } @@ -138,37 +147,36 @@ export const StepCreateForm: FC = React.memo( async function startTransform() { setLoading(true); - try { - const resp = await api.startTransforms([{ id: transformId }]); - if (typeof resp === 'object' && resp !== null && resp[transformId]?.success === true) { - toastNotifications.addSuccess( - i18n.translate('xpack.transform.stepCreateForm.startTransformSuccessMessage', { - defaultMessage: 'Request to start transform {transformId} acknowledged.', - values: { transformId }, - }) - ); - setStarted(true); - setLoading(false); - } else { - const errorMessage = - typeof resp === 'object' && resp !== null && resp[transformId]?.success === false - ? resp[transformId].error - : resp; - throw new Error(errorMessage); - } - } catch (e) { - toastNotifications.addDanger({ - title: i18n.translate('xpack.transform.stepCreateForm.startTransformErrorMessage', { - defaultMessage: 'An error occurred starting the transform {transformId}:', + const resp = await api.startTransforms([{ id: transformId }]); + + if (isStartTransformsResponseSchema(resp) && resp[transformId]?.success === true) { + toastNotifications.addSuccess( + i18n.translate('xpack.transform.stepCreateForm.startTransformSuccessMessage', { + defaultMessage: 'Request to start transform {transformId} acknowledged.', values: { transformId }, - }), - text: toMountPoint( - - ), - }); - setStarted(false); + }) + ); + setStarted(true); setLoading(false); + return; } + + const errorMessage = + isStartTransformsResponseSchema(resp) && resp[transformId]?.success === false + ? resp[transformId].error + : resp; + + toastNotifications.addDanger({ + title: i18n.translate('xpack.transform.stepCreateForm.startTransformErrorMessage', { + defaultMessage: 'An error occurred starting the transform {transformId}:', + values: { transformId }, + }), + text: toMountPoint( + + ), + }); + setStarted(false); + setLoading(false); } async function createAndStartTransform() { @@ -250,27 +258,30 @@ export const StepCreateForm: FC = React.memo( // wrapping in function so we can keep the interval id in local scope function startProgressBar() { const interval = setInterval(async () => { - try { - const stats = await api.getTransformsStats(transformId); - if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) { - const percent = - getTransformProgress({ - id: transformConfig.id, - config: transformConfig, - stats: stats.transforms[0], - }) || 0; - setProgressPercentComplete(percent); - if (percent >= 100) { - clearInterval(interval); - } + const stats = await api.getTransformStats(transformId); + + if ( + isGetTransformsStatsResponseSchema(stats) && + Array.isArray(stats.transforms) && + stats.transforms.length > 0 + ) { + const percent = + getTransformProgress({ + id: transformConfig.id, + config: transformConfig, + stats: stats.transforms[0], + }) || 0; + setProgressPercentComplete(percent); + if (percent >= 100) { + clearInterval(interval); } - } catch (e) { + } else { toastNotifications.addDanger({ title: i18n.translate('xpack.transform.stepCreateForm.progressErrorMessage', { defaultMessage: 'An error occurred getting the progress percentage:', }), text: toMountPoint( - + ), }); clearInterval(interval); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts index fba703b1540f9..1523a0d9a89f9 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts @@ -6,19 +6,21 @@ import { isEqual } from 'lodash'; +import { Dictionary } from '../../../../../../../common/types/common'; +import { PivotSupportedAggs } from '../../../../../../../common/types/pivot_aggs'; +import { TransformPivotConfig } from '../../../../../../../common/types/transform'; + import { matchAllQuery, PivotAggsConfig, PivotAggsConfigDict, PivotGroupByConfig, PivotGroupByConfigDict, - TransformPivotConfig, PIVOT_SUPPORTED_GROUP_BY_AGGS, } from '../../../../../common'; -import { Dictionary } from '../../../../../../../common/types/common'; import { StepDefineExposedState } from './types'; -import { getAggConfigFromEsAgg, PivotSupportedAggs } from '../../../../../common/pivot_aggs'; +import { getAggConfigFromEsAgg } from '../../../../../common/pivot_aggs'; export function applyTransformConfigToDefineState( state: StepDefineExposedState, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx index 9d3ab44aa5708..d59f99192621c 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx @@ -10,6 +10,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { debounce } from 'lodash'; import { useUpdateEffect } from 'react-use'; import { i18n } from '@kbn/i18n'; +import { isEsSearchResponse } from '../../../../../../../../../common/api_schemas/type_guards'; import { useApi } from '../../../../../../../hooks'; import { CreateTransformWizardContext } from '../../../../wizard/wizard'; import { FilterAggConfigTerm } from '../types'; @@ -55,22 +56,24 @@ export const FilterTermForm: FilterAggConfigTerm['aggTypeConfig']['FilterAggForm }, }; - try { - const response = await api.esSearch(esSearchRequest); - setOptions( - response.aggregations.field_values.buckets.map( - (value: { key: string; doc_count: number }) => ({ label: value.key }) - ) - ); - } catch (e) { + const response = await api.esSearch(esSearchRequest); + + setIsLoading(false); + + if (!isEsSearchResponse(response)) { toastNotifications.addWarning( i18n.translate('xpack.transform.agg.popoverForm.filerAgg.term.errorFetchSuggestions', { defaultMessage: 'Unable to fetch suggestions', }) ); + return; } - setIsLoading(false); + setOptions( + response.aggregations.field_values.buckets.map( + (value: { key: string; doc_count: number }) => ({ label: value.key }) + ) + ); }, 600), [selectedField] ); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_form_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_form_config.ts index 2839c1181c333..5575e6d814daf 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_form_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_form_config.ts @@ -5,11 +5,11 @@ */ import { - PIVOT_SUPPORTED_AGGS, - PivotAggsConfigBase, - PivotAggsConfigWithUiBase, PivotSupportedAggs, -} from '../../../../../common/pivot_aggs'; + PIVOT_SUPPORTED_AGGS, +} from '../../../../../../../common/types/pivot_aggs'; + +import { PivotAggsConfigBase, PivotAggsConfigWithUiBase } from '../../../../../common/pivot_aggs'; import { getFilterAggConfig } from './filter_agg/config'; /** diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_name_conflict_toast_messages.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_name_conflict_toast_messages.ts index 57f9397089f1d..03cbf2e358736 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_name_conflict_toast_messages.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_agg_name_conflict_toast_messages.ts @@ -6,7 +6,8 @@ import { i18n } from '@kbn/i18n'; -import { AggName, PivotAggsConfigDict, PivotGroupByConfigDict } from '../../../../../common'; +import { AggName } from '../../../../../../../common/types/aggregations'; +import { PivotAggsConfigDict, PivotGroupByConfigDict } from '../../../../../common'; export function getAggNameConflictToastMessages( aggName: AggName, @@ -36,7 +37,7 @@ export function getAggNameConflictToastMessages( // check the new aggName against existing aggs and groupbys const aggNameSplit = aggName.split('.'); let aggNameCheck: string; - aggNameSplit.forEach((aggNamePart) => { + aggNameSplit.forEach((aggNamePart: string) => { aggNameCheck = aggNameCheck === undefined ? aggNamePart : `${aggNameCheck}.${aggNamePart}`; if (aggList[aggNameCheck] !== undefined || groupByList[aggNameCheck] !== undefined) { conflicts.push( diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_aggregation_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_aggregation_config.ts index 460164c9afe73..14c03aebe892a 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_aggregation_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_aggregation_config.ts @@ -4,13 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +import { EsFieldName } from '../../../../../../../common/types/fields'; import { - EsFieldName, - PERCENTILES_AGG_DEFAULT_PERCENTS, + PivotSupportedAggs, PIVOT_SUPPORTED_AGGS, +} from '../../../../../../../common/types/pivot_aggs'; +import { + PERCENTILES_AGG_DEFAULT_PERCENTS, PivotAggsConfigWithUiSupport, } from '../../../../../common'; -import { PivotSupportedAggs } from '../../../../../common/pivot_aggs'; import { getFilterAggConfig } from './filter_agg/config'; /** diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_group_by_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_group_by_config.ts index 712a745ff6e77..657e8c935b875 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_group_by_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_group_by_config.ts @@ -4,11 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - EsFieldName, - GroupByConfigWithUiSupport, - PIVOT_SUPPORTED_GROUP_BY_AGGS, -} from '../../../../../common'; +import { EsFieldName } from '../../../../../../../common/types/fields'; + +import { GroupByConfigWithUiSupport, PIVOT_SUPPORTED_GROUP_BY_AGGS } from '../../../../../common'; export function getDefaultGroupByConfig( aggName: string, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts index 56fde98cd4c71..955982aae6007 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts @@ -6,7 +6,9 @@ import { KBN_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public'; -import { EsFieldName, PivotAggsConfigDict, PivotGroupByConfigDict } from '../../../../../common'; +import { EsFieldName } from '../../../../../../../common/types/fields'; + +import { PivotAggsConfigDict, PivotGroupByConfigDict } from '../../../../../common'; import { SavedSearchQuery } from '../../../../../hooks/use_search_items'; import { QUERY_LANGUAGE } from './constants'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_pivot_editor.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_pivot_editor.ts index 2e92114286599..41b84f04db852 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_pivot_editor.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_pivot_editor.ts @@ -8,13 +8,13 @@ import { useEffect, useState } from 'react'; import { useXJsonMode } from '../../../../../../../../../../src/plugins/es_ui_shared/static/ace_x_json/hooks'; -import { PreviewRequestBody } from '../../../../../common'; +import { PostTransformsPreviewRequestSchema } from '../../../../../../../common/api_schemas/transforms'; import { StepDefineExposedState } from '../common'; export const useAdvancedPivotEditor = ( defaults: StepDefineExposedState, - previewRequest: PreviewRequestBody + previewRequest: PostTransformsPreviewRequestSchema ) => { const stringifiedPivotConfig = JSON.stringify(previewRequest.pivot, null, 2); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_source_editor.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_source_editor.ts index 1ea8a45248fb9..3f930711b970a 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_source_editor.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_source_editor.ts @@ -6,13 +6,13 @@ import { useState } from 'react'; -import { PreviewRequestBody } from '../../../../../common'; +import { PostTransformsPreviewRequestSchema } from '../../../../../../../common/api_schemas/transforms'; import { StepDefineExposedState } from '../common'; export const useAdvancedSourceEditor = ( defaults: StepDefineExposedState, - previewRequest: PreviewRequestBody + previewRequest: PostTransformsPreviewRequestSchema ) => { const stringifiedSourceConfig = JSON.stringify(previewRequest.source.query, null, 2); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts index d35d567fc8469..90b28f0e305a5 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts @@ -6,11 +6,11 @@ import { useCallback, useMemo, useState } from 'react'; +import { AggName } from '../../../../../../../common/types/aggregations'; import { dictionaryToArray } from '../../../../../../../common/types/common'; import { useToastNotifications } from '../../../../../app_dependencies'; import { - AggName, DropDownLabel, PivotAggsConfig, PivotAggsConfigDict, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts index f5980ae2243d3..7c10201fc3a6e 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts @@ -6,7 +6,7 @@ import { useEffect } from 'react'; -import { getPreviewRequestBody } from '../../../../../common'; +import { getPreviewTransformRequestBody } from '../../../../../common'; import { getDefaultStepDefineState } from '../common'; @@ -26,7 +26,7 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi const searchBar = useSearchBar(defaults, indexPattern); const pivotConfig = usePivotConfig(defaults, indexPattern); - const previewRequest = getPreviewRequestBody( + const previewRequest = getPreviewTransformRequestBody( indexPattern.title, searchBar.state.pivotQuery, pivotConfig.state.pivotGroupByArr, @@ -41,7 +41,7 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi useEffect(() => { if (!advancedSourceEditor.state.isAdvancedSourceEditorEnabled) { - const previewRequestUpdate = getPreviewRequestBody( + const previewRequestUpdate = getPreviewTransformRequestBody( indexPattern.title, searchBar.state.pivotQuery, pivotConfig.state.pivotGroupByArr, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.test.tsx index 8c919a5185d7e..986ac0a212e8a 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.test.tsx @@ -15,10 +15,11 @@ import { coreMock } from '../../../../../../../../../src/core/public/mocks'; import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; const startMock = coreMock.createStart(); +import { PIVOT_SUPPORTED_AGGS } from '../../../../../../common/types/pivot_aggs'; + import { PivotAggsConfigDict, PivotGroupByConfigDict, - PIVOT_SUPPORTED_AGGS, PIVOT_SUPPORTED_GROUP_BY_AGGS, } from '../../../../common'; import { SearchItems } from '../../../../hooks/use_search_items'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx index e0b350542a8f8..10f473074b4d7 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx @@ -22,6 +22,9 @@ import { EuiText, } from '@elastic/eui'; +import { PivotAggDict } from '../../../../../../common/types/pivot_aggs'; +import { PivotGroupByDict } from '../../../../../../common/types/pivot_group_by'; + import { DataGrid } from '../../../../../shared_imports'; import { @@ -30,10 +33,8 @@ import { } from '../../../../common/data_grid'; import { - getPreviewRequestBody, - PivotAggDict, + getPreviewTransformRequestBody, PivotAggsConfigDict, - PivotGroupByDict, PivotGroupByConfigDict, PivotSupportedGroupByAggs, PivotAggsConfig, @@ -87,7 +88,7 @@ export const StepDefineForm: FC = React.memo((props) => { toastNotifications, }; - const previewRequest = getPreviewRequestBody( + const previewRequest = getPreviewTransformRequestBody( indexPattern.title, pivotQuery, pivotGroupByArr, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.test.tsx index dc3d950938c9e..f8a060e0007b8 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.test.tsx @@ -7,10 +7,11 @@ import React from 'react'; import { render, wait } from '@testing-library/react'; +import { PIVOT_SUPPORTED_AGGS } from '../../../../../../common/types/pivot_aggs'; + import { PivotAggsConfig, PivotGroupByConfig, - PIVOT_SUPPORTED_AGGS, PIVOT_SUPPORTED_GROUP_BY_AGGS, } from '../../../../common'; import { SearchItems } from '../../../../hooks/use_search_items'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx index 414f6e37504da..fa4f8a7e09690 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx @@ -18,7 +18,7 @@ import { useToastNotifications } from '../../../../app_dependencies'; import { getPivotQuery, getPivotPreviewDevConsoleStatement, - getPreviewRequestBody, + getPreviewTransformRequestBody, isDefaultQuery, isMatchAllQuery, } from '../../../../common'; @@ -44,7 +44,7 @@ export const StepDefineSummary: FC = ({ const pivotGroupByArr = dictionaryToArray(groupByList); const pivotQuery = getPivotQuery(searchQuery); - const previewRequest = getPreviewRequestBody( + const previewRequest = getPreviewTransformRequestBody( searchItems.indexPattern.title, pivotQuery, pivotGroupByArr, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx index 85f4065e8c069..43d4f11cffc9d 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx @@ -11,24 +11,28 @@ import { i18n } from '@kbn/i18n'; import { EuiLink, EuiSwitch, EuiFieldText, EuiForm, EuiFormRow, EuiSelect } from '@elastic/eui'; import { KBN_FIELD_TYPES } from '../../../../../../../../../src/plugins/data/common'; - import { toMountPoint } from '../../../../../../../../../src/plugins/kibana_react/public'; -import { TransformId } from '../../../../../../common'; + +import { + isEsIndices, + isPostTransformsPreviewResponseSchema, +} from '../../../../../../common/api_schemas/type_guards'; +import { TransformId, TransformPivotConfig } from '../../../../../../common/types/transform'; import { isValidIndexName } from '../../../../../../common/utils/es_utils'; import { getErrorMessage } from '../../../../../../common/utils/errors'; import { useAppDependencies, useToastNotifications } from '../../../../app_dependencies'; import { ToastNotificationText } from '../../../../components'; +import { isHttpFetchError } from '../../../../common/request'; import { useDocumentationLinks } from '../../../../hooks/use_documentation_links'; import { SearchItems } from '../../../../hooks/use_search_items'; import { useApi } from '../../../../hooks/use_api'; import { StepDetailsTimeField } from './step_details_time_field'; import { getPivotQuery, - getPreviewRequestBody, + getPreviewTransformRequestBody, isTransformIdValid, - TransformPivotConfig, } from '../../../../common'; import { EsIndexName, IndexPatternTitle } from './common'; import { delayValidator } from '../../../../common/validators'; @@ -48,10 +52,12 @@ export interface StepDetailsExposedState { indexPatternDateField?: string | undefined; } +const defaultContinuousModeDelay = '60s'; + export function getDefaultStepDetailsState(): StepDetailsExposedState { return { continuousModeDateField: '', - continuousModeDelay: '60s', + continuousModeDelay: defaultContinuousModeDelay, createIndexPattern: true, isContinuousModeEnabled: false, transformId: '', @@ -72,7 +78,7 @@ export function applyTransformConfigToDetailsState( const time = transformConfig.sync?.time; if (time !== undefined) { state.continuousModeDateField = time.field; - state.continuousModeDelay = time.delay; + state.continuousModeDelay = time?.delay ?? defaultContinuousModeDelay; state.isContinuousModeEnabled = true; } } @@ -137,19 +143,20 @@ export const StepDetailsForm: FC = React.memo( useEffect(() => { // use an IIFE to avoid returning a Promise to useEffect. (async function () { - try { - const { searchQuery, groupByList, aggList } = stepDefineState; - const pivotAggsArr = dictionaryToArray(aggList); - const pivotGroupByArr = dictionaryToArray(groupByList); - const pivotQuery = getPivotQuery(searchQuery); - const previewRequest = getPreviewRequestBody( - searchItems.indexPattern.title, - pivotQuery, - pivotGroupByArr, - pivotAggsArr - ); - - const transformPreview = await api.getTransformsPreview(previewRequest); + const { searchQuery, groupByList, aggList } = stepDefineState; + const pivotAggsArr = dictionaryToArray(aggList); + const pivotGroupByArr = dictionaryToArray(groupByList); + const pivotQuery = getPivotQuery(searchQuery); + const previewRequest = getPreviewTransformRequestBody( + searchItems.indexPattern.title, + pivotQuery, + pivotGroupByArr, + pivotAggsArr + ); + + const transformPreview = await api.getTransformsPreview(previewRequest); + + if (isPostTransformsPreviewResponseSchema(transformPreview)) { const properties = transformPreview.generated_dest_index.mappings.properties; const datetimeColumns: string[] = Object.keys(properties).filter( (col) => properties[col].type === 'date' @@ -157,43 +164,46 @@ export const StepDetailsForm: FC = React.memo( setPreviewDateColumns(datetimeColumns); setIndexPatternDateField(datetimeColumns[0]); - } catch (e) { + } else { toastNotifications.addDanger({ title: i18n.translate('xpack.transform.stepDetailsForm.errorGettingTransformPreview', { - defaultMessage: 'An error occurred getting transform preview', + defaultMessage: 'An error occurred fetching the transform preview', }), text: toMountPoint( - + ), }); } - try { - setTransformIds( - (await api.getTransforms()).transforms.map( - (transform: TransformPivotConfig) => transform.id - ) - ); - } catch (e) { + const resp = await api.getTransforms(); + + if (isHttpFetchError(resp)) { toastNotifications.addDanger({ title: i18n.translate('xpack.transform.stepDetailsForm.errorGettingTransformList', { defaultMessage: 'An error occurred getting the existing transform IDs:', }), text: toMountPoint( - + ), }); + } else { + setTransformIds(resp.transforms.map((transform: TransformPivotConfig) => transform.id)); } - try { - setIndexNames((await api.getIndices()).map((index) => index.name)); - } catch (e) { + const indices = await api.getEsIndices(); + + if (isEsIndices(indices)) { + setIndexNames(indices.map((index) => index.name)); + } else { toastNotifications.addDanger({ title: i18n.translate('xpack.transform.stepDetailsForm.errorGettingIndexNames', { defaultMessage: 'An error occurred getting the existing index names:', }), text: toMountPoint( - + ), }); } diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx index 806dcbfa75604..0ca018972cac9 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx @@ -10,7 +10,9 @@ import { i18n } from '@kbn/i18n'; import { EuiSteps, EuiStepStatus } from '@elastic/eui'; -import { getCreateRequestBody, TransformPivotConfig } from '../../../../common'; +import { TransformPivotConfig } from '../../../../../../common/types/transform'; + +import { getCreateTransformRequestBody } from '../../../../common'; import { SearchItems } from '../../../../hooks/use_search_items'; import { @@ -149,7 +151,7 @@ export const Wizard: FC = React.memo(({ cloneConfig, searchItems }) } }, []); - const transformConfig = getCreateRequestBody( + const transformConfig = getCreateTransformRequestBody( indexPattern.title, stepDefineState, stepDetailsState diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx index d8ab72f15c59c..75868fb8fcabd 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx @@ -7,7 +7,7 @@ import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiToolTip } from '@elastic/eui'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants'; import { createCapabilityFailureMessage } from '../../../../lib/authorization'; import { TransformListRow } from '../../../../common'; @@ -18,8 +18,8 @@ export const deleteActionNameText = i18n.translate( } ); -const transformCanNotBeDeleted = (item: TransformListRow) => - ![TRANSFORM_STATE.STOPPED, TRANSFORM_STATE.FAILED].includes(item.stats.state); +const transformCanNotBeDeleted = (i: TransformListRow) => + !([TRANSFORM_STATE.STOPPED, TRANSFORM_STATE.FAILED] as TransformState[]).includes(i.stats.state); export const isDeleteActionDisabled = (items: TransformListRow[], forceDisable: boolean) => { const disabled = items.some(transformCanNotBeDeleted); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx index e573709fa6e63..7e8e099b69f82 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx @@ -6,7 +6,7 @@ import React, { useContext, useMemo, useState } from 'react'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { TransformListAction, TransformListRow } from '../../../../common'; import { useDeleteIndexAndTargetIndex, useDeleteTransforms } from '../../../../hooks'; @@ -55,7 +55,16 @@ export const useDeleteAction = (forceDisable: boolean) => { const forceDelete = isBulkAction ? shouldForceDelete : items[0] && items[0] && items[0].stats.state === TRANSFORM_STATE.FAILED; - deleteTransforms(items, shouldDeleteDestIndex, shouldDeleteDestIndexPattern, forceDelete); + + deleteTransforms({ + transformsInfo: items.map((i) => ({ + id: i.config.id, + state: i.stats.state, + })), + deleteDestIndex: shouldDeleteDestIndex, + deleteDestIndexPattern: shouldDeleteDestIndexPattern, + forceDelete, + }); }; const openModal = (newItems: TransformListRow[]) => { diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_edit/use_edit_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_edit/use_edit_action.tsx index 1fe20f1acae5a..192ff7ac74c57 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_edit/use_edit_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_edit/use_edit_action.tsx @@ -6,7 +6,9 @@ import React, { useContext, useMemo, useState } from 'react'; -import { TransformListAction, TransformListRow, TransformPivotConfig } from '../../../../common'; +import { TransformPivotConfig } from '../../../../../../common/types/transform'; + +import { TransformListAction, TransformListRow } from '../../../../common'; import { AuthorizationContext } from '../../../../lib/authorization'; import { editActionNameText, EditActionName } from './edit_action_name'; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx index 191df0c16cba0..ca1c90b9b8fae 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx @@ -8,7 +8,7 @@ import React, { FC, useContext } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiToolTip } from '@elastic/eui'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { createCapabilityFailureMessage, diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx index 8d6a4376c55b3..96af60778d6a4 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx @@ -6,7 +6,7 @@ import React, { useContext, useMemo, useState } from 'react'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { AuthorizationContext } from '../../../../lib/authorization'; import { TransformListAction, TransformListRow } from '../../../../common'; @@ -27,7 +27,7 @@ export const useStartAction = (forceDisable: boolean) => { const startAndCloseModal = () => { setModalVisible(false); - startTransforms(items); + startTransforms(items.map((i) => ({ id: i.id }))); }; const openModal = (newItems: TransformListRow[]) => { diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx index e1ea82cb371e8..4ec30faa4d76b 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx @@ -8,7 +8,7 @@ import React, { FC, useContext } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiToolTip } from '@elastic/eui'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { TransformListRow } from '../../../../common'; import { diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx index e0a7e0b489ab6..4c872114a82ab 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx @@ -6,7 +6,7 @@ import React, { useCallback, useContext, useMemo } from 'react'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { AuthorizationContext } from '../../../../lib/authorization'; import { TransformListAction, TransformListRow } from '../../../../common'; @@ -20,9 +20,10 @@ export const useStopAction = (forceDisable: boolean) => { const stopTransforms = useStopTransforms(); - const clickHandler = useCallback((item: TransformListRow) => stopTransforms([item]), [ - stopTransforms, - ]); + const clickHandler = useCallback( + (i: TransformListRow) => stopTransforms([{ id: i.id, state: i.stats.state }]), + [stopTransforms] + ); const action: TransformListAction = useMemo( () => ({ diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_flyout.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_flyout.tsx index 735a059e57e14..f9cdac51b6582 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_flyout.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_flyout.tsx @@ -23,13 +23,12 @@ import { EuiTitle, } from '@elastic/eui'; +import { isPostTransformsUpdateResponseSchema } from '../../../../../../common/api_schemas/type_guards'; +import { TransformPivotConfig } from '../../../../../../common/types/transform'; + import { getErrorMessage } from '../../../../../../common/utils/errors'; -import { - refreshTransformList$, - TransformPivotConfig, - REFRESH_TRANSFORM_LIST_STATE, -} from '../../../../common'; +import { refreshTransformList$, REFRESH_TRANSFORM_LIST_STATE } from '../../../../common'; import { useToastNotifications } from '../../../../app_dependencies'; import { useApi } from '../../../../hooks/use_api'; @@ -58,19 +57,21 @@ export const EditTransformFlyout: FC = ({ closeFlyout, const requestConfig = applyFormFieldsToTransformConfig(config, state.formFields); const transformId = config.id; - try { - await api.updateTransform(transformId, requestConfig); - toastNotifications.addSuccess( - i18n.translate('xpack.transform.transformList.editTransformSuccessMessage', { - defaultMessage: 'Transform {transformId} updated.', - values: { transformId }, - }) - ); - closeFlyout(); - refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.REFRESH); - } catch (e) { - setErrorMessage(getErrorMessage(e)); + const resp = await api.updateTransform(transformId, requestConfig); + + if (!isPostTransformsUpdateResponseSchema(resp)) { + setErrorMessage(getErrorMessage(resp)); + return; } + + toastNotifications.addSuccess( + i18n.translate('xpack.transform.transformList.editTransformSuccessMessage', { + defaultMessage: 'Transform {transformId} updated.', + values: { transformId }, + }) + ); + closeFlyout(); + refreshTransformList$.next(REFRESH_TRANSFORM_LIST_STATE.REFRESH); } const isUpdateButtonDisabled = !state.isFormValid || !state.isFormTouched; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.test.ts b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.test.ts index 4a8b26b601ae2..12e60c2af5556 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.test.ts +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { TransformPivotConfig } from '../../../../common'; +import { TransformPivotConfig } from '../../../../../../common/types/transform'; import { applyFormFieldsToTransformConfig, @@ -86,9 +86,7 @@ describe('Transform: applyFormFieldsToTransformConfig()', () => { }); test('should include previously nonexisting attributes', () => { - const transformConfigMock = getTransformConfigMock(); - delete transformConfigMock.description; - delete transformConfigMock.frequency; + const { description, frequency, ...transformConfigMock } = getTransformConfigMock(); const updateConfig = applyFormFieldsToTransformConfig(transformConfigMock, { description: getDescriptionFieldMock('the-new-description'), diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts index 649db51e6ea78..d622a7e9cc040 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts @@ -9,7 +9,8 @@ import { useReducer } from 'react'; import { i18n } from '@kbn/i18n'; -import { TransformPivotConfig } from '../../../../common'; +import { PostTransformsUpdateRequestSchema } from '../../../../../../common/api_schemas/update_transforms'; +import { TransformPivotConfig } from '../../../../../../common/types/transform'; // A Validator function takes in a value to check and returns an array of error messages. // If no messages (empty array) get returned, the value is valid. @@ -118,53 +119,35 @@ interface Action { value: string; } -// Some attributes can have a value of `null` to trigger -// a reset to the default value, or in the case of `docs_per_second` -// `null` is used to disable throttling. -interface UpdateTransformPivotConfig { - description: string; - frequency: string; - settings: { - docs_per_second: number | null; - }; -} - // Takes in the form configuration and returns a // request object suitable to be sent to the // transform update API endpoint. export const applyFormFieldsToTransformConfig = ( config: TransformPivotConfig, { description, docsPerSecond, frequency }: EditTransformFlyoutFieldsState -): Partial => { - const updateConfig: Partial = {}; - - // set the values only if they changed from the default - // and actually differ from the previous value. - if ( - !(config.frequency === undefined && frequency.value === '') && - config.frequency !== frequency.value - ) { - updateConfig.frequency = frequency.value; - } - - if ( - !(config.description === undefined && description.value === '') && - config.description !== description.value - ) { - updateConfig.description = description.value; - } - +): PostTransformsUpdateRequestSchema => { // if the input field was left empty, // fall back to the default value of `null` // which will disable throttling. const docsPerSecondFormValue = docsPerSecond.value !== '' ? parseInt(docsPerSecond.value, 10) : null; const docsPerSecondConfigValue = config.settings?.docs_per_second ?? null; - if (docsPerSecondFormValue !== docsPerSecondConfigValue) { - updateConfig.settings = { docs_per_second: docsPerSecondFormValue }; - } - return updateConfig; + return { + // set the values only if they changed from the default + // and actually differ from the previous value. + ...(!(config.frequency === undefined && frequency.value === '') && + config.frequency !== frequency.value + ? { frequency: frequency.value } + : {}), + ...(!(config.description === undefined && description.value === '') && + config.description !== description.value + ? { description: description.value } + : {}), + ...(docsPerSecondFormValue !== docsPerSecondConfigValue + ? { settings: { docs_per_second: docsPerSecondFormValue } } + : {}), + }; }; // Takes in a transform configuration and returns diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/common.test.ts b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/common.test.ts index 11e4dc3dfa2b8..f6708f7c36f26 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/common.test.ts +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/common.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; import mockTransformListRow from '../../../../common/__mocks__/transform_list_row.json'; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_messages_pane.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_messages_pane.tsx index 08545c288ba96..02bad50dc0dfd 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_messages_pane.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_messages_pane.tsx @@ -9,11 +9,15 @@ import React, { useState } from 'react'; import { EuiSpacer, EuiBasicTable } from '@elastic/eui'; // @ts-ignore import { formatDate } from '@elastic/eui/lib/services/format'; -import { i18n } from '@kbn/i18n'; import theme from '@elastic/eui/dist/eui_theme_light.json'; + +import { i18n } from '@kbn/i18n'; + +import { isGetTransformsAuditMessagesResponseSchema } from '../../../../../../common/api_schemas/type_guards'; +import { TransformMessage } from '../../../../../../common/types/messages'; + import { useApi } from '../../../../hooks/use_api'; import { JobIcon } from '../../../../components/job_icon'; -import { TransformMessage } from '../../../../../../common/types/messages'; import { useRefreshTransformList } from '../../../../common'; const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss'; @@ -36,25 +40,16 @@ export const ExpandedRowMessagesPane: React.FC = ({ transformId }) => { let concurrentLoads = 0; return async function getMessages() { - try { - concurrentLoads++; - - if (concurrentLoads > 1) { - return; - } + concurrentLoads++; - setIsLoading(true); - const messagesResp = await api.getTransformAuditMessages(transformId); - setIsLoading(false); - setMessages(messagesResp as any[]); + if (concurrentLoads > 1) { + return; + } - concurrentLoads--; + setIsLoading(true); + const messagesResp = await api.getTransformAuditMessages(transformId); - if (concurrentLoads > 0) { - concurrentLoads = 0; - getMessages(); - } - } catch (error) { + if (!isGetTransformsAuditMessagesResponseSchema(messagesResp)) { setIsLoading(false); setErrorMessage( i18n.translate( @@ -64,6 +59,17 @@ export const ExpandedRowMessagesPane: React.FC = ({ transformId }) => { } ) ); + return; + } + + setIsLoading(false); + setMessages(messagesResp as any[]); + + concurrentLoads--; + + if (concurrentLoads > 0) { + concurrentLoads = 0; + getMessages(); } }; }; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx index a917fc73ad8fb..87d9a25dababd 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx @@ -6,10 +6,11 @@ import React, { useMemo, FC } from 'react'; +import { TransformPivotConfig } from '../../../../../../common/types/transform'; import { DataGrid } from '../../../../../shared_imports'; import { useToastNotifications } from '../../../../app_dependencies'; -import { getPivotQuery, TransformPivotConfig } from '../../../../common'; +import { getPivotQuery } from '../../../../common'; import { usePivotData } from '../../../../hooks/use_pivot_data'; import { SearchItems } from '../../../../hooks/use_search_items'; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx index dad0f0e5ee282..12836c0a18ce2 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx @@ -23,7 +23,7 @@ import { EuiTitle, } from '@elastic/eui'; -import { TransformId } from '../../../../../../common'; +import { TransformId } from '../../../../../../common/types/transform'; import { useRefreshTransformList, @@ -189,7 +189,11 @@ export const TransformList: FC = ({ ,
- stopTransforms(transformSelection)}> + + stopTransforms(transformSelection.map((t) => ({ id: t.id, state: t.stats.state }))) + } + >
, diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar.tsx index fab591f881310..fdcb9ba5f0aff 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar.tsx @@ -16,8 +16,8 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { TermClause, FieldClause, Value } from './common'; -import { TRANSFORM_STATE } from '../../../../../../common'; -import { TRANSFORM_MODE, TransformListRow } from '../../../../common'; +import { TRANSFORM_MODE, TRANSFORM_STATE } from '../../../../../../common/constants'; +import { TransformListRow } from '../../../../common'; import { getTaskStateBadge } from './use_columns'; const filters: SearchFilterConfig[] = [ diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx index bce01b954c83e..313668d4c5180 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx @@ -7,9 +7,9 @@ import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; -import { TRANSFORM_STATE } from '../../../../../../common'; +import { TRANSFORM_MODE, TRANSFORM_STATE } from '../../../../../../common/constants'; -import { TRANSFORM_MODE, TransformListRow } from '../../../../common'; +import { TransformListRow } from '../../../../common'; import { StatsBar, TransformStatsBarStats } from '../stats_bar'; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx index d2d8c7084941d..040e502ce4888 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx @@ -22,24 +22,21 @@ import { RIGHT_ALIGNMENT, } from '@elastic/eui'; -import { TransformId, TRANSFORM_STATE } from '../../../../../../common'; +import { TransformId } from '../../../../../../common/types/transform'; +import { TransformStats } from '../../../../../../common/types/transform_stats'; +import { TRANSFORM_STATE } from '../../../../../../common/constants'; -import { - getTransformProgress, - TransformListRow, - TransformStats, - TRANSFORM_LIST_COLUMN, -} from '../../../../common'; +import { getTransformProgress, TransformListRow, TRANSFORM_LIST_COLUMN } from '../../../../common'; import { useActions } from './use_actions'; -enum STATE_COLOR { - aborting = 'warning', - failed = 'danger', - indexing = 'primary', - started = 'primary', - stopped = 'hollow', - stopping = 'hollow', -} +const STATE_COLOR = { + aborting: 'warning', + failed: 'danger', + indexing: 'primary', + started: 'primary', + stopped: 'hollow', + stopping: 'hollow', +} as const; export const getTaskStateBadge = ( state: TransformStats['state'], diff --git a/x-pack/plugins/transform/public/shared_imports.ts b/x-pack/plugins/transform/public/shared_imports.ts index 196df250b7a3d..4737787dbd9ee 100644 --- a/x-pack/plugins/transform/public/shared_imports.ts +++ b/x-pack/plugins/transform/public/shared_imports.ts @@ -27,7 +27,6 @@ export { DataGrid, EsSorting, RenderCellValue, - SearchResponse7, UseDataGridReturnType, UseIndexDataReturnType, INDEX_STATUS, diff --git a/x-pack/plugins/transform/server/README.md b/x-pack/plugins/transform/server/README.md new file mode 100644 index 0000000000000..1142c1fea094d --- /dev/null +++ b/x-pack/plugins/transform/server/README.md @@ -0,0 +1,19 @@ +# Transform Kibana API routes + +This folder contains Transform API routes in Kibana. + +Each route handler requires [apiDoc](https://github.com/apidoc/apidoc) annotations in order +to generate documentation. +The [apidoc-markdown](https://github.com/rigwild/apidoc-markdown) package is also required in order to generate the markdown. + +There are custom parser and worker (`x-pack/plugins/transform/server/routes/apidoc_scripts`) to process api schemas for each documentation entry. It's written with typescript so make sure all the scripts in the folder are compiled before executing `apidoc` command. + +Make sure you have run `yarn kbn bootstrap` to get all requires dev dependencies. Then execute the following command from the transform plugin folder: +``` +yarn run apiDocs +``` +It compiles all the required scripts and generates the documentation both in HTML and Markdown formats. + + +It will create a new directory `routes_doc` (next to the `routes` folder) which contains the documentation in HTML format +as well as `Transform_API.md` file. diff --git a/x-pack/plugins/transform/server/routes/api/error_utils.ts b/x-pack/plugins/transform/server/routes/api/error_utils.ts index 5a479e4f429f6..269cd28c4bda6 100644 --- a/x-pack/plugins/transform/server/routes/api/error_utils.ts +++ b/x-pack/plugins/transform/server/routes/api/error_utils.ts @@ -10,11 +10,8 @@ import { i18n } from '@kbn/i18n'; import { ResponseError, CustomHttpResponseOptions } from 'src/core/server'; -import { - TransformEndpointRequest, - TransformEndpointResult, - DeleteTransformEndpointResult, -} from '../../../common'; +import { CommonResponseStatusSchema, TransformIdsSchema } from '../../../common/api_schemas/common'; +import { DeleteTransformsResponseSchema } from '../../../common/api_schemas/delete_transforms'; const REQUEST_TIMEOUT = 'RequestTimeout'; @@ -23,9 +20,9 @@ export function isRequestTimeout(error: any) { } interface Params { - results: TransformEndpointResult | DeleteTransformEndpointResult; + results: CommonResponseStatusSchema | DeleteTransformsResponseSchema; id: string; - items: TransformEndpointRequest[]; + items: TransformIdsSchema; action: string; } @@ -63,7 +60,7 @@ export function fillResultsWithTimeouts({ results, id, items, action }: Params) }, }; - const newResults: TransformEndpointResult | DeleteTransformEndpointResult = {}; + const newResults: CommonResponseStatusSchema | DeleteTransformsResponseSchema = {}; return items.reduce((accumResults, currentVal) => { if (results[currentVal.id] === undefined) { diff --git a/x-pack/plugins/transform/server/routes/api/field_histograms.ts b/x-pack/plugins/transform/server/routes/api/field_histograms.ts index 2642040c4cd0d..88352ec4af129 100644 --- a/x-pack/plugins/transform/server/routes/api/field_histograms.ts +++ b/x-pack/plugins/transform/server/routes/api/field_histograms.ts @@ -11,40 +11,49 @@ import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; +import { + indexPatternTitleSchema, + IndexPatternTitleSchema, +} from '../../../common/api_schemas/common'; +import { + fieldHistogramsRequestSchema, + FieldHistogramsRequestSchema, +} from '../../../common/api_schemas/field_histograms'; import { getHistogramsForFields } from '../../shared_imports'; import { RouteDependencies } from '../../types'; import { addBasePath } from '../index'; import { wrapError } from './error_utils'; -import { fieldHistogramsSchema, indexPatternTitleSchema, IndexPatternTitleSchema } from './schema'; export function registerFieldHistogramsRoutes({ router, license }: RouteDependencies) { - router.post( + router.post( { path: addBasePath('field_histograms/{indexPatternTitle}'), validate: { params: indexPatternTitleSchema, - body: fieldHistogramsSchema, + body: fieldHistogramsRequestSchema, }, }, - license.guardApiRoute(async (ctx, req, res) => { - const { indexPatternTitle } = req.params as IndexPatternTitleSchema; - const { query, fields, samplerShardSize } = req.body; + license.guardApiRoute( + async (ctx, req, res) => { + const { indexPatternTitle } = req.params; + const { query, fields, samplerShardSize } = req.body; - try { - const resp = await getHistogramsForFields( - ctx.core.elasticsearch.client, - indexPatternTitle, - query, - fields, - samplerShardSize - ); + try { + const resp = await getHistogramsForFields( + ctx.core.elasticsearch.client, + indexPatternTitle, + query, + fields, + samplerShardSize + ); - return res.ok({ body: resp }); - } catch (e) { - return res.customError(wrapError(wrapEsError(e))); + return res.ok({ body: resp }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } } - }) + ) ); } diff --git a/x-pack/plugins/transform/server/routes/api/privileges.ts b/x-pack/plugins/transform/server/routes/api/privileges.ts index 2b7b0544a8bf9..605cbde356fdf 100644 --- a/x-pack/plugins/transform/server/routes/api/privileges.ts +++ b/x-pack/plugins/transform/server/routes/api/privileges.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { APP_CLUSTER_PRIVILEGES, APP_INDEX_PRIVILEGES } from '../../../common/constants'; -import { Privileges } from '../../../common'; +import { Privileges } from '../../../common/types/privileges'; import { RouteDependencies } from '../../types'; import { addBasePath } from '../index'; diff --git a/x-pack/plugins/transform/server/routes/api/schema.ts b/x-pack/plugins/transform/server/routes/api/schema.ts deleted file mode 100644 index 8aadef81b221b..0000000000000 --- a/x-pack/plugins/transform/server/routes/api/schema.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { schema } from '@kbn/config-schema'; - -export const fieldHistogramsSchema = schema.object({ - /** Query to match documents in the index. */ - query: schema.any(), - /** The fields to return histogram data. */ - fields: schema.arrayOf(schema.any()), - /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ - samplerShardSize: schema.number(), -}); - -export const indexPatternTitleSchema = schema.object({ - /** Title of the index pattern for which to return stats. */ - indexPatternTitle: schema.string(), -}); - -export interface IndexPatternTitleSchema { - indexPatternTitle: string; -} - -export const schemaTransformId = { - params: schema.object({ - transformId: schema.string(), - }), -}; - -export interface SchemaTransformId { - transformId: string; -} - -export const deleteTransformSchema = schema.object({ - /** - * Delete Transform & Destination Index - */ - transformsInfo: schema.arrayOf( - schema.object({ - id: schema.string(), - state: schema.maybe(schema.string()), - }) - ), - deleteDestIndex: schema.maybe(schema.boolean()), - deleteDestIndexPattern: schema.maybe(schema.boolean()), - forceDelete: schema.maybe(schema.boolean()), -}); diff --git a/x-pack/plugins/transform/server/routes/api/transforms.ts b/x-pack/plugins/transform/server/routes/api/transforms.ts index efbe813db5e67..c02bc06ad6060 100644 --- a/x-pack/plugins/transform/server/routes/api/transforms.ts +++ b/x-pack/plugins/transform/server/routes/api/transforms.ts @@ -14,22 +14,47 @@ import { import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; +import { TRANSFORM_STATE } from '../../../common/constants'; +import { TransformId } from '../../../common/types/transform'; import { - TransformEndpointRequest, - TransformEndpointResult, - TransformId, - TRANSFORM_STATE, - DeleteTransformEndpointRequest, - DeleteTransformStatus, - ResultData, -} from '../../../common'; + transformIdParamSchema, + ResponseStatus, + TransformIdParamSchema, +} from '../../../common/api_schemas/common'; +import { + deleteTransformsRequestSchema, + DeleteTransformsRequestSchema, + DeleteTransformsResponseSchema, +} from '../../../common/api_schemas/delete_transforms'; +import { + startTransformsRequestSchema, + StartTransformsRequestSchema, + StartTransformsResponseSchema, +} from '../../../common/api_schemas/start_transforms'; +import { + stopTransformsRequestSchema, + StopTransformsRequestSchema, + StopTransformsResponseSchema, +} from '../../../common/api_schemas/stop_transforms'; +import { + postTransformsUpdateRequestSchema, + PostTransformsUpdateRequestSchema, + PostTransformsUpdateResponseSchema, +} from '../../../common/api_schemas/update_transforms'; +import { + GetTransformsResponseSchema, + postTransformsPreviewRequestSchema, + PostTransformsPreviewRequestSchema, + putTransformsRequestSchema, + PutTransformsRequestSchema, + PutTransformsResponseSchema, +} from '../../../common/api_schemas/transforms'; import { RouteDependencies } from '../../types'; import { addBasePath } from '../index'; import { isRequestTimeout, fillResultsWithTimeouts, wrapError } from './error_utils'; -import { deleteTransformSchema, schemaTransformId, SchemaTransformId } from './schema'; import { registerTransformsAuditMessagesRoutes } from './transforms_audit_messages'; import { IIndexPattern } from '../../../../../../src/plugins/data/common/index_patterns'; @@ -47,6 +72,16 @@ interface StopOptions { export function registerTransformsRoutes(routeDependencies: RouteDependencies) { const { router, license } = routeDependencies; + /** + * @apiGroup Transforms + * + * @api {get} /api/transform/transforms Get transforms + * @apiName GetTransforms + * @apiDescription Returns transforms + * + * @apiSchema (params) jobAuditMessagesJobIdSchema + * @apiSchema (query) jobAuditMessagesQuerySchema + */ router.get( { path: addBasePath('transforms'), validate: false }, license.guardApiRoute(async (ctx, req, res) => { @@ -62,16 +97,24 @@ export function registerTransformsRoutes(routeDependencies: RouteDependencies) { } }) ); - router.get( + + /** + * @apiGroup Transforms + * + * @api {get} /api/transform/transforms/:transformId Get transform + * @apiName GetTransform + * @apiDescription Returns a single transform + * + * @apiSchema (params) transformIdParamSchema + */ + router.get( { path: addBasePath('transforms/{transformId}'), - validate: schemaTransformId, + validate: { params: transformIdParamSchema }, }, - license.guardApiRoute(async (ctx, req, res) => { - const { transformId } = req.params as SchemaTransformId; - const options = { - ...(transformId !== undefined ? { transformId } : {}), - }; + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params; + const options = transformId !== undefined ? { transformId } : {}; try { const transforms = await getTransforms( options, @@ -83,6 +126,14 @@ export function registerTransformsRoutes(routeDependencies: RouteDependencies) { } }) ); + + /** + * @apiGroup Transforms + * + * @api {get} /api/transform/transforms/_stats Get transforms stats + * @apiName GetTransformsStats + * @apiDescription Returns transforms stats + */ router.get( { path: addBasePath('transforms/_stats'), validate: false }, license.guardApiRoute(async (ctx, req, res) => { @@ -98,13 +149,23 @@ export function registerTransformsRoutes(routeDependencies: RouteDependencies) { } }) ); - router.get( + + /** + * @apiGroup Transforms + * + * @api {get} /api/transform/transforms/:transformId/_stats Get transform stats + * @apiName GetTransformStats + * @apiDescription Returns stats for a single transform + * + * @apiSchema (params) transformIdParamSchema + */ + router.get( { path: addBasePath('transforms/{transformId}/_stats'), - validate: schemaTransformId, + validate: { params: transformIdParamSchema }, }, - license.guardApiRoute(async (ctx, req, res) => { - const { transformId } = req.params as SchemaTransformId; + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params; const options = { ...(transformId !== undefined ? { transformId } : {}), }; @@ -120,134 +181,198 @@ export function registerTransformsRoutes(routeDependencies: RouteDependencies) { }) ); registerTransformsAuditMessagesRoutes(routeDependencies); - router.put( + + /** + * @apiGroup Transforms + * + * @api {put} /api/transform/transforms/:transformId Put transform + * @apiName PutTransform + * @apiDescription Creates a transform + * + * @apiSchema (params) transformIdParamSchema + * @apiSchema (body) putTransformsRequestSchema + */ + router.put( { path: addBasePath('transforms/{transformId}'), validate: { - ...schemaTransformId, - body: schema.maybe(schema.any()), + params: transformIdParamSchema, + body: putTransformsRequestSchema, }, }, - license.guardApiRoute(async (ctx, req, res) => { - const { transformId } = req.params as SchemaTransformId; - - const response: { - transformsCreated: Array<{ transform: string }>; - errors: any[]; - } = { - transformsCreated: [], - errors: [], - }; + license.guardApiRoute( + async (ctx, req, res) => { + const { transformId } = req.params; - await ctx - .transform!.dataClient.callAsCurrentUser('transform.createTransform', { - body: req.body, - transformId, - }) - .then(() => response.transformsCreated.push({ transform: transformId })) - .catch((e) => - response.errors.push({ - id: transformId, - error: wrapEsError(e), + const response: PutTransformsResponseSchema = { + transformsCreated: [], + errors: [], + }; + + await ctx + .transform!.dataClient.callAsCurrentUser('transform.createTransform', { + body: req.body, + transformId, }) - ); + .then(() => response.transformsCreated.push({ transform: transformId })) + .catch((e) => + response.errors.push({ + id: transformId, + error: wrapEsError(e), + }) + ); - return res.ok({ body: response }); - }) + return res.ok({ body: response }); + } + ) ); - router.post( + + /** + * @apiGroup Transforms + * + * @api {post} /api/transform/transforms/:transformId/_update Post transform update + * @apiName PostTransformUpdate + * @apiDescription Updates a transform + * + * @apiSchema (params) transformIdParamSchema + * @apiSchema (body) postTransformsUpdateRequestSchema + */ + router.post( { path: addBasePath('transforms/{transformId}/_update'), validate: { - ...schemaTransformId, - body: schema.maybe(schema.any()), + params: transformIdParamSchema, + body: postTransformsUpdateRequestSchema, }, }, - license.guardApiRoute(async (ctx, req, res) => { - const { transformId } = req.params as SchemaTransformId; + license.guardApiRoute( + async (ctx, req, res) => { + const { transformId } = req.params; - try { - return res.ok({ - body: await ctx.transform!.dataClient.callAsCurrentUser('transform.updateTransform', { - body: req.body, - transformId, - }), - }); - } catch (e) { - return res.customError(wrapError(e)); + try { + return res.ok({ + body: (await ctx.transform!.dataClient.callAsCurrentUser('transform.updateTransform', { + body: req.body, + transformId, + })) as PostTransformsUpdateResponseSchema, + }); + } catch (e) { + return res.customError(wrapError(e)); + } } - }) + ) ); - router.post( + + /** + * @apiGroup Transforms + * + * @api {post} /api/transform/delete_transforms Post delete transforms + * @apiName DeleteTransforms + * @apiDescription Deletes transforms + * + * @apiSchema (body) deleteTransformsRequestSchema + */ + router.post( { path: addBasePath('delete_transforms'), validate: { - body: deleteTransformSchema, + body: deleteTransformsRequestSchema, }, }, - license.guardApiRoute(async (ctx, req, res) => { - const { - transformsInfo, - deleteDestIndex, - deleteDestIndexPattern, - forceDelete, - } = req.body as DeleteTransformEndpointRequest; - - try { - const body = await deleteTransforms( - transformsInfo, - deleteDestIndex, - deleteDestIndexPattern, - forceDelete, - ctx, - license, - res - ); - - if (body && body.status) { - if (body.status === 404) { - return res.notFound(); - } - if (body.status === 403) { - return res.forbidden(); + license.guardApiRoute( + async (ctx, req, res) => { + try { + const body = await deleteTransforms(req.body, ctx, res); + + if (body && body.status) { + if (body.status === 404) { + return res.notFound(); + } + if (body.status === 403) { + return res.forbidden(); + } } - } - return res.ok({ - body, - }); - } catch (e) { - return res.customError(wrapError(wrapEsError(e))); + return res.ok({ + body, + }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } } - }) + ) ); - router.post( + + /** + * @apiGroup Transforms + * + * @api {post} /api/transform/transforms/_preview Preview transform + * @apiName PreviewTransform + * @apiDescription Previews transform + * + * @apiSchema (body) postTransformsPreviewRequestSchema + */ + router.post( { path: addBasePath('transforms/_preview'), validate: { - body: schema.maybe(schema.any()), + body: postTransformsPreviewRequestSchema, }, }, - license.guardApiRoute(previewTransformHandler) + license.guardApiRoute( + previewTransformHandler + ) ); - router.post( + + /** + * @apiGroup Transforms + * + * @api {post} /api/transform/start_transforms Start transforms + * @apiName PostStartTransforms + * @apiDescription Starts transform + * + * @apiSchema (body) startTransformsRequestSchema + */ + router.post( { path: addBasePath('start_transforms'), validate: { - body: schema.maybe(schema.any()), + body: startTransformsRequestSchema, }, }, - license.guardApiRoute(startTransformsHandler) + license.guardApiRoute( + startTransformsHandler + ) ); - router.post( + + /** + * @apiGroup Transforms + * + * @api {post} /api/transform/stop_transforms Stop transforms + * @apiName PostStopTransforms + * @apiDescription Stops transform + * + * @apiSchema (body) stopTransformsRequestSchema + */ + router.post( { path: addBasePath('stop_transforms'), validate: { - body: schema.maybe(schema.any()), + body: stopTransformsRequestSchema, }, }, - license.guardApiRoute(stopTransformsHandler) + license.guardApiRoute(stopTransformsHandler) ); + + /** + * @apiGroup Transforms + * + * @api {post} /api/transform/es_search Transform ES Search Proxy + * @apiName PostTransformEsSearchProxy + * @apiDescription ES Search Proxy + * + * @apiSchema (body) any + */ router.post( { path: addBasePath('es_search'), @@ -267,7 +392,10 @@ export function registerTransformsRoutes(routeDependencies: RouteDependencies) { ); } -const getTransforms = async (options: { transformId?: string }, callAsCurrentUser: CallCluster) => { +const getTransforms = async ( + options: { transformId?: string }, + callAsCurrentUser: CallCluster +): Promise => { return await callAsCurrentUser('transform.getTransforms', options); }; @@ -294,22 +422,25 @@ async function deleteDestIndexPatternById( } async function deleteTransforms( - transformsInfo: TransformEndpointRequest[], - deleteDestIndex: boolean | undefined, - deleteDestIndexPattern: boolean | undefined, - shouldForceDelete: boolean = false, + reqBody: DeleteTransformsRequestSchema, ctx: RequestHandlerContext, - license: RouteDependencies['license'], response: KibanaResponseFactory ) { - const results: Record = {}; + const { transformsInfo } = reqBody; + + // Cast possible undefineds as booleans + const deleteDestIndex = !!reqBody.deleteDestIndex; + const deleteDestIndexPattern = !!reqBody.deleteDestIndexPattern; + const shouldForceDelete = !!reqBody.forceDelete; + + const results: DeleteTransformsResponseSchema = {}; for (const transformInfo of transformsInfo) { let destinationIndex: string | undefined; - const transformDeleted: ResultData = { success: false }; - const destIndexDeleted: ResultData = { success: false }; - const destIndexPatternDeleted: ResultData = { + const transformDeleted: ResponseStatus = { success: false }; + const destIndexDeleted: ResponseStatus = { success: false }; + const destIndexPatternDeleted: ResponseStatus = { success: false, }; const transformId = transformInfo.id; @@ -405,7 +536,11 @@ async function deleteTransforms( return results; } -const previewTransformHandler: RequestHandler = async (ctx, req, res) => { +const previewTransformHandler: RequestHandler< + undefined, + undefined, + PostTransformsPreviewRequestSchema +> = async (ctx, req, res) => { try { return res.ok({ body: await ctx.transform!.dataClient.callAsCurrentUser('transform.getTransformsPreview', { @@ -417,8 +552,12 @@ const previewTransformHandler: RequestHandler = async (ctx, req, res) => { } }; -const startTransformsHandler: RequestHandler = async (ctx, req, res) => { - const transformsInfo = req.body as TransformEndpointRequest[]; +const startTransformsHandler: RequestHandler< + undefined, + undefined, + StartTransformsRequestSchema +> = async (ctx, req, res) => { + const transformsInfo = req.body; try { return res.ok({ @@ -430,15 +569,15 @@ const startTransformsHandler: RequestHandler = async (ctx, req, res) => { }; async function startTransforms( - transformsInfo: TransformEndpointRequest[], + transformsInfo: StartTransformsRequestSchema, callAsCurrentUser: CallCluster ) { - const results: TransformEndpointResult = {}; + const results: StartTransformsResponseSchema = {}; for (const transformInfo of transformsInfo) { const transformId = transformInfo.id; try { - await callAsCurrentUser('transform.startTransform', { transformId } as SchemaTransformId); + await callAsCurrentUser('transform.startTransform', { transformId }); results[transformId] = { success: true }; } catch (e) { if (isRequestTimeout(e)) { @@ -455,8 +594,12 @@ async function startTransforms( return results; } -const stopTransformsHandler: RequestHandler = async (ctx, req, res) => { - const transformsInfo = req.body as TransformEndpointRequest[]; +const stopTransformsHandler: RequestHandler< + undefined, + undefined, + StopTransformsRequestSchema +> = async (ctx, req, res) => { + const transformsInfo = req.body; try { return res.ok({ @@ -468,10 +611,10 @@ const stopTransformsHandler: RequestHandler = async (ctx, req, res) => { }; async function stopTransforms( - transformsInfo: TransformEndpointRequest[], + transformsInfo: StopTransformsRequestSchema, callAsCurrentUser: CallCluster ) { - const results: TransformEndpointResult = {}; + const results: StopTransformsResponseSchema = {}; for (const transformInfo of transformsInfo) { const transformId = transformInfo.id; diff --git a/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts b/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts index 722a3f52376b4..f01b2bdb73fd5 100644 --- a/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts +++ b/x-pack/plugins/transform/server/routes/api/transforms_audit_messages.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { AuditMessage } from '../../../common/types/messages'; +import { transformIdParamSchema, TransformIdParamSchema } from '../../../common/api_schemas/common'; +import { AuditMessage, TransformMessage } from '../../../common/types/messages'; import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; import { RouteDependencies } from '../../types'; @@ -12,7 +13,6 @@ import { RouteDependencies } from '../../types'; import { addBasePath } from '../index'; import { wrapError } from './error_utils'; -import { schemaTransformId, SchemaTransformId } from './schema'; const ML_DF_NOTIFICATION_INDEX_PATTERN = '.transform-notifications-read'; const SIZE = 500; @@ -22,10 +22,22 @@ interface BoolQuery { } export function registerTransformsAuditMessagesRoutes({ router, license }: RouteDependencies) { - router.get( - { path: addBasePath('transforms/{transformId}/messages'), validate: schemaTransformId }, - license.guardApiRoute(async (ctx, req, res) => { - const { transformId } = req.params as SchemaTransformId; + /** + * @apiGroup Transforms Audit Messages + * + * @api {get} /api/transform/transforms/:transformId/messages Transforms Messages + * @apiName GetTransformsMessages + * @apiDescription Get transforms audit messages + * + * @apiSchema (params) transformIdParamSchema + */ + router.get( + { + path: addBasePath('transforms/{transformId}/messages'), + validate: { params: transformIdParamSchema }, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { transformId } = req.params; // search for audit messages, // transformId is optional. without it, all transforms will be listed. @@ -77,7 +89,7 @@ export function registerTransformsAuditMessagesRoutes({ router, license }: Route }, }); - let messages = []; + let messages: TransformMessage[] = []; if (resp.hits.total !== 0) { messages = resp.hits.hits.map((hit: AuditMessage) => hit._source); messages.reverse(); diff --git a/x-pack/plugins/transform/server/routes/apidoc.json b/x-pack/plugins/transform/server/routes/apidoc.json new file mode 100644 index 0000000000000..ce76b5b302f93 --- /dev/null +++ b/x-pack/plugins/transform/server/routes/apidoc.json @@ -0,0 +1,21 @@ +{ + "name": "transform_kibana_api", + "version": "7.10.0", + "description": "This is the documentation of the REST API provided by the Transform Kibana plugin. Each API is experimental and can include breaking changes in any version.", + "title": "Transform Kibana API", + "order": [ + "GetTransforms", + "GetTransform", + "GetTransformsStats", + "GetTransformStats", + "PutTransform", + "PostTransformUpdate", + "DeleteTransforms", + "PreviewTransform", + "PostStartTransforms", + "PostStopTransforms", + "PostTransformEsSearchProxy", + "DeleteDataFrameAnalytics", + "GetTransformsMessages" + ] +} diff --git a/x-pack/plugins/transform/server/services/license.ts b/x-pack/plugins/transform/server/services/license.ts index 1a2768999fdc4..bacf9724a6253 100644 --- a/x-pack/plugins/transform/server/services/license.ts +++ b/x-pack/plugins/transform/server/services/license.ts @@ -62,12 +62,12 @@ export class License { }); } - guardApiRoute(handler: RequestHandler) { + guardApiRoute(handler: RequestHandler) { const license = this; return function licenseCheck( ctx: RequestHandlerContext, - request: KibanaRequest, + request: KibanaRequest, response: KibanaResponseFactory ): IKibanaResponse | Promise> { const licenseStatus = license.getStatus(); diff --git a/x-pack/plugins/transform/tsconfig.json b/x-pack/plugins/transform/tsconfig.json new file mode 100644 index 0000000000000..6f83eb665f830 --- /dev/null +++ b/x-pack/plugins/transform/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "../../tsconfig.json", +} diff --git a/x-pack/run_functional_tests.sh b/x-pack/run_functional_tests.sh deleted file mode 100755 index e94f283ea0394..0000000000000 --- a/x-pack/run_functional_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -export TEST_KIBANA_URL="http://elastic:mlqa_admin@localhost:5601" -export TEST_ES_URL="http://elastic:mlqa_admin@localhost:9200" -node ../scripts/functional_test_runner --include-tag walterra diff --git a/x-pack/test/api_integration/apis/transform/common.ts b/x-pack/test/api_integration/apis/transform/common.ts new file mode 100644 index 0000000000000..1a48ee987bc77 --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/common.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import type { PutTransformsRequestSchema } from '../../../../plugins/transform/common/api_schemas/transforms'; + +export async function asyncForEach(array: any[], callback: Function) { + for (let index = 0; index < array.length; index++) { + await callback(array[index], index, array); + } +} + +export function generateDestIndex(transformId: string): string { + return `user-${transformId}`; +} + +export function generateTransformConfig(transformId: string): PutTransformsRequestSchema { + const destinationIndex = generateDestIndex(transformId); + + return { + source: { index: ['ft_farequote'] }, + pivot: { + group_by: { airline: { terms: { field: 'airline' } } }, + aggregations: { '@timestamp.value_count': { value_count: { field: '@timestamp' } } }, + }, + dest: { index: destinationIndex }, + }; +} diff --git a/x-pack/test/api_integration/apis/transform/delete_transforms.ts b/x-pack/test/api_integration/apis/transform/delete_transforms.ts index 7f01d2741ad15..41b2bffb1f0ad 100644 --- a/x-pack/test/api_integration/apis/transform/delete_transforms.ts +++ b/x-pack/test/api_integration/apis/transform/delete_transforms.ts @@ -4,41 +4,28 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { TransformEndpointRequest } from '../../../../plugins/transform/common'; -import { FtrProviderContext } from '../../ftr_provider_context'; + +import { DeleteTransformsRequestSchema } from '../../../../plugins/transform/common/api_schemas/delete_transforms'; +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; import { USER } from '../../../functional/services/transform/security_common'; -async function asyncForEach(array: any[], callback: Function) { - for (let index = 0; index < array.length; index++) { - await callback(array[index], index, array); - } -} +import { FtrProviderContext } from '../../ftr_provider_context'; + +import { asyncForEach, generateDestIndex, generateTransformConfig } from './common'; export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const supertest = getService('supertestWithoutAuth'); const transform = getService('transform'); - function generateDestIndex(transformId: string): string { - return `user-${transformId}`; + async function createTransform(transformId: string) { + const config = generateTransformConfig(transformId); + await transform.api.createTransform(transformId, config); } - async function createTransform(transformId: string, destinationIndex: string) { - const config = { - id: transformId, - source: { index: ['farequote-*'] }, - pivot: { - group_by: { airline: { terms: { field: 'airline' } } }, - aggregations: { '@timestamp.value_count': { value_count: { field: '@timestamp' } } }, - }, - dest: { index: destinationIndex }, - }; - - await transform.api.createTransform(config); - } - - describe('delete_transforms', function () { + describe('/api/transform/delete_transforms', function () { before(async () => { await esArchiver.loadIfNeeded('ml/farequote'); await transform.testResources.setKibanaTimeZoneToUTC(); @@ -49,11 +36,11 @@ export default ({ getService }: FtrProviderContext) => { }); describe('single transform deletion', function () { - const transformId = 'test1'; + const transformId = 'transform-test-delete'; const destinationIndex = generateDestIndex(transformId); beforeEach(async () => { - await createTransform(transformId, destinationIndex); + await createTransform(transformId); await transform.api.createIndices(destinationIndex); }); @@ -62,7 +49,9 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete transform by transformId', async () => { - const transformsInfo: TransformEndpointRequest[] = [{ id: transformId }]; + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [{ id: transformId, state: TRANSFORM_STATE.STOPPED }], + }; const { body } = await supertest .post(`/api/transform/delete_transforms`) .auth( @@ -70,9 +59,7 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - }) + .send(reqBody) .expect(200); expect(body[transformId].transformDeleted.success).to.eql(true); @@ -83,7 +70,9 @@ export default ({ getService }: FtrProviderContext) => { }); it('should return 403 for unauthorized user', async () => { - const transformsInfo: TransformEndpointRequest[] = [{ id: transformId }]; + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [{ id: transformId, state: TRANSFORM_STATE.STOPPED }], + }; await supertest .post(`/api/transform/delete_transforms`) .auth( @@ -91,9 +80,7 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - }) + .send(reqBody) .expect(403); await transform.api.waitForTransformToExist(transformId); await transform.api.waitForIndicesToExist(destinationIndex); @@ -102,7 +89,9 @@ export default ({ getService }: FtrProviderContext) => { describe('single transform deletion with invalid transformId', function () { it('should return 200 with error in response if invalid transformId', async () => { - const transformsInfo: TransformEndpointRequest[] = [{ id: 'invalid_transform_id' }]; + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [{ id: 'invalid_transform_id', state: TRANSFORM_STATE.STOPPED }], + }; const { body } = await supertest .post(`/api/transform/delete_transforms`) .auth( @@ -110,9 +99,7 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - }) + .send(reqBody) .expect(200); expect(body.invalid_transform_id.transformDeleted.success).to.eql(false); expect(body.invalid_transform_id.transformDeleted).to.have.property('error'); @@ -120,15 +107,17 @@ export default ({ getService }: FtrProviderContext) => { }); describe('bulk deletion', function () { - const transformsInfo: TransformEndpointRequest[] = [ - { id: 'bulk_delete_test_1' }, - { id: 'bulk_delete_test_2' }, - ]; - const destinationIndices = transformsInfo.map((d) => generateDestIndex(d.id)); + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [ + { id: 'bulk_delete_test_1', state: TRANSFORM_STATE.STOPPED }, + { id: 'bulk_delete_test_2', state: TRANSFORM_STATE.STOPPED }, + ], + }; + const destinationIndices = reqBody.transformsInfo.map((d) => generateDestIndex(d.id)); beforeEach(async () => { - await asyncForEach(transformsInfo, async ({ id }: { id: string }, idx: number) => { - await createTransform(id, destinationIndices[idx]); + await asyncForEach(reqBody.transformsInfo, async ({ id }: { id: string }, idx: number) => { + await createTransform(id); await transform.api.createIndices(destinationIndices[idx]); }); }); @@ -147,13 +136,11 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - }) + .send(reqBody) .expect(200); await asyncForEach( - transformsInfo, + reqBody.transformsInfo, async ({ id: transformId }: { id: string }, idx: number) => { expect(body[transformId].transformDeleted.success).to.eql(true); expect(body[transformId].destIndexDeleted.success).to.eql(false); @@ -174,16 +161,16 @@ export default ({ getService }: FtrProviderContext) => { ) .set(COMMON_REQUEST_HEADERS) .send({ + ...reqBody, transformsInfo: [ - { id: transformsInfo[0].id }, - { id: invalidTransformId }, - { id: transformsInfo[1].id }, + ...reqBody.transformsInfo, + { id: invalidTransformId, state: TRANSFORM_STATE.STOPPED }, ], }) .expect(200); await asyncForEach( - transformsInfo, + reqBody.transformsInfo, async ({ id: transformId }: { id: string }, idx: number) => { expect(body[transformId].transformDeleted.success).to.eql(true); expect(body[transformId].destIndexDeleted.success).to.eql(false); @@ -203,7 +190,7 @@ export default ({ getService }: FtrProviderContext) => { const destinationIndex = generateDestIndex(transformId); before(async () => { - await createTransform(transformId, destinationIndex); + await createTransform(transformId); await transform.api.createIndices(destinationIndex); }); @@ -212,7 +199,10 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete transform and destination index', async () => { - const transformsInfo: TransformEndpointRequest[] = [{ id: transformId }]; + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [{ id: transformId, state: TRANSFORM_STATE.STOPPED }], + deleteDestIndex: true, + }; const { body } = await supertest .post(`/api/transform/delete_transforms`) .auth( @@ -220,10 +210,7 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - deleteDestIndex: true, - }) + .send(reqBody) .expect(200); expect(body[transformId].transformDeleted.success).to.eql(true); @@ -239,7 +226,7 @@ export default ({ getService }: FtrProviderContext) => { const destinationIndex = generateDestIndex(transformId); before(async () => { - await createTransform(transformId, destinationIndex); + await createTransform(transformId); await transform.api.createIndices(destinationIndex); await transform.testResources.createIndexPatternIfNeeded(destinationIndex); }); @@ -250,7 +237,11 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete transform and destination index pattern', async () => { - const transformsInfo: TransformEndpointRequest[] = [{ id: transformId }]; + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [{ id: transformId, state: TRANSFORM_STATE.STOPPED }], + deleteDestIndex: false, + deleteDestIndexPattern: true, + }; const { body } = await supertest .post(`/api/transform/delete_transforms`) .auth( @@ -258,11 +249,7 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - deleteDestIndex: false, - deleteDestIndexPattern: true, - }) + .send(reqBody) .expect(200); expect(body[transformId].transformDeleted.success).to.eql(true); @@ -279,7 +266,7 @@ export default ({ getService }: FtrProviderContext) => { const destinationIndex = generateDestIndex(transformId); before(async () => { - await createTransform(transformId, destinationIndex); + await createTransform(transformId); await transform.api.createIndices(destinationIndex); await transform.testResources.createIndexPatternIfNeeded(destinationIndex); }); @@ -290,7 +277,11 @@ export default ({ getService }: FtrProviderContext) => { }); it('should delete transform, destination index, & destination index pattern', async () => { - const transformsInfo: TransformEndpointRequest[] = [{ id: transformId }]; + const reqBody: DeleteTransformsRequestSchema = { + transformsInfo: [{ id: transformId, state: TRANSFORM_STATE.STOPPED }], + deleteDestIndex: true, + deleteDestIndexPattern: true, + }; const { body } = await supertest .post(`/api/transform/delete_transforms`) .auth( @@ -298,11 +289,7 @@ export default ({ getService }: FtrProviderContext) => { transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) ) .set(COMMON_REQUEST_HEADERS) - .send({ - transformsInfo, - deleteDestIndex: true, - deleteDestIndexPattern: true, - }) + .send(reqBody) .expect(200); expect(body[transformId].transformDeleted.success).to.eql(true); diff --git a/x-pack/test/api_integration/apis/transform/index.ts b/x-pack/test/api_integration/apis/transform/index.ts index 93a951a55ece1..ef08883534d10 100644 --- a/x-pack/test/api_integration/apis/transform/index.ts +++ b/x-pack/test/api_integration/apis/transform/index.ts @@ -28,5 +28,11 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { }); loadTestFile(require.resolve('./delete_transforms')); + loadTestFile(require.resolve('./start_transforms')); + loadTestFile(require.resolve('./stop_transforms')); + loadTestFile(require.resolve('./transforms')); + loadTestFile(require.resolve('./transforms_preview')); + loadTestFile(require.resolve('./transforms_stats')); + loadTestFile(require.resolve('./transforms_update')); }); } diff --git a/x-pack/test/api_integration/apis/transform/start_transforms.ts b/x-pack/test/api_integration/apis/transform/start_transforms.ts new file mode 100644 index 0000000000000..288a3caae390e --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/start_transforms.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import { StartTransformsRequestSchema } from '../../../../plugins/transform/common/api_schemas/start_transforms'; +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + +import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; +import { USER } from '../../../functional/services/transform/security_common'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +import { asyncForEach, generateDestIndex, generateTransformConfig } from './common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const transform = getService('transform'); + + async function createTransform(transformId: string) { + const config = generateTransformConfig(transformId); + await transform.api.createTransform(transformId, config); + } + + describe('/api/transform/start_transforms', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await transform.testResources.setKibanaTimeZoneToUTC(); + }); + + describe('single transform start', function () { + const transformId = 'transform-test-start'; + const destinationIndex = generateDestIndex(transformId); + + beforeEach(async () => { + await createTransform(transformId); + }); + + afterEach(async () => { + await transform.api.cleanTransformIndices(); + await transform.api.deleteIndices(destinationIndex); + }); + + it('should start the transform by transformId', async () => { + const reqBody: StartTransformsRequestSchema = [{ id: transformId }]; + const { body } = await supertest + .post(`/api/transform/start_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(body[transformId].success).to.eql(true); + expect(typeof body[transformId].error).to.eql('undefined'); + await transform.api.waitForBatchTransformToComplete(transformId); + await transform.api.waitForIndicesToExist(destinationIndex); + }); + + it('should return 200 with success:false for unauthorized user', async () => { + const reqBody: StartTransformsRequestSchema = [{ id: transformId }]; + const { body } = await supertest + .post(`/api/transform/start_transforms`) + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(body[transformId].success).to.eql(false); + expect(typeof body[transformId].error).to.eql('string'); + + await transform.api.waitForTransformState(transformId, TRANSFORM_STATE.STOPPED); + await transform.api.waitForIndicesNotToExist(destinationIndex); + }); + }); + + describe('single transform start with invalid transformId', function () { + it('should return 200 with error in response if invalid transformId', async () => { + const reqBody: StartTransformsRequestSchema = [{ id: 'invalid_transform_id' }]; + const { body } = await supertest + .post(`/api/transform/start_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(body.invalid_transform_id.success).to.eql(false); + expect(body.invalid_transform_id).to.have.property('error'); + }); + }); + + describe('bulk start', function () { + const reqBody: StartTransformsRequestSchema = [ + { id: 'bulk_start_test_1' }, + { id: 'bulk_start_test_2' }, + ]; + const destinationIndices = reqBody.map((d) => generateDestIndex(d.id)); + + beforeEach(async () => { + await asyncForEach(reqBody, async ({ id }: { id: string }, idx: number) => { + await createTransform(id); + }); + }); + + afterEach(async () => { + await transform.api.cleanTransformIndices(); + await asyncForEach(destinationIndices, async (destinationIndex: string) => { + await transform.api.deleteIndices(destinationIndex); + }); + }); + + it('should start multiple transforms by transformIds', async () => { + const { body } = await supertest + .post(`/api/transform/start_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + await asyncForEach(reqBody, async ({ id: transformId }: { id: string }, idx: number) => { + expect(body[transformId].success).to.eql(true); + await transform.api.waitForBatchTransformToComplete(transformId); + await transform.api.waitForIndicesToExist(destinationIndices[idx]); + }); + }); + + it('should start multiple transforms by transformIds, even if one of the transformIds is invalid', async () => { + const invalidTransformId = 'invalid_transform_id'; + const { body } = await supertest + .post(`/api/transform/start_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send([{ id: reqBody[0].id }, { id: invalidTransformId }, { id: reqBody[1].id }]) + .expect(200); + + await asyncForEach(reqBody, async ({ id: transformId }: { id: string }, idx: number) => { + expect(body[transformId].success).to.eql(true); + await transform.api.waitForBatchTransformToComplete(transformId); + await transform.api.waitForIndicesToExist(destinationIndices[idx]); + }); + + expect(body[invalidTransformId].success).to.eql(false); + expect(body[invalidTransformId]).to.have.property('error'); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/transform/stop_transforms.ts b/x-pack/test/api_integration/apis/transform/stop_transforms.ts new file mode 100644 index 0000000000000..4f30db0794ea4 --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/stop_transforms.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import type { PutTransformsRequestSchema } from '../../../../plugins/transform/common/api_schemas/transforms'; +import type { StopTransformsRequestSchema } from '../../../../plugins/transform/common/api_schemas/stop_transforms'; +import { isStopTransformsResponseSchema } from '../../../../plugins/transform/common/api_schemas/type_guards'; + +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + +import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; +import { USER } from '../../../functional/services/transform/security_common'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +import { asyncForEach, generateDestIndex, generateTransformConfig } from './common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const transform = getService('transform'); + + async function createAndRunTransform(transformId: string) { + // to be able to test stopping transforms, + // we create a slow continuous transform + // so it doesn't stop automatically. + const config: PutTransformsRequestSchema = { + ...generateTransformConfig(transformId), + settings: { + docs_per_second: 10, + max_page_search_size: 10, + }, + sync: { + time: { field: '@timestamp' }, + }, + }; + + await transform.api.createAndRunTransform(transformId, config); + } + + describe('/api/transform/stop_transforms', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await transform.testResources.setKibanaTimeZoneToUTC(); + }); + + describe('single transform stop', function () { + const transformId = 'transform-test-stop'; + const destinationIndex = generateDestIndex(transformId); + + beforeEach(async () => { + await createAndRunTransform(transformId); + }); + + afterEach(async () => { + await transform.api.cleanTransformIndices(); + await transform.api.deleteIndices(destinationIndex); + }); + + it('should stop the transform by transformId', async () => { + const reqBody: StopTransformsRequestSchema = [ + { id: transformId, state: TRANSFORM_STATE.STARTED }, + ]; + const { body } = await supertest + .post(`/api/transform/stop_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(isStopTransformsResponseSchema(body)).to.eql(true); + expect(body[transformId].success).to.eql(true); + expect(typeof body[transformId].error).to.eql('undefined'); + await transform.api.waitForTransformState(transformId, TRANSFORM_STATE.STOPPED); + await transform.api.waitForIndicesToExist(destinationIndex); + }); + + it('should return 200 with success:false for unauthorized user', async () => { + const reqBody: StopTransformsRequestSchema = [ + { id: transformId, state: TRANSFORM_STATE.STARTED }, + ]; + const { body } = await supertest + .post(`/api/transform/stop_transforms`) + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(isStopTransformsResponseSchema(body)).to.eql(true); + expect(body[transformId].success).to.eql(false); + expect(typeof body[transformId].error).to.eql('string'); + + await transform.api.waitForTransformStateNotToBe(transformId, TRANSFORM_STATE.STOPPED); + await transform.api.waitForIndicesToExist(destinationIndex); + }); + }); + + describe('single transform stop with invalid transformId', function () { + it('should return 200 with error in response if invalid transformId', async () => { + const reqBody: StopTransformsRequestSchema = [ + { id: 'invalid_transform_id', state: TRANSFORM_STATE.STARTED }, + ]; + const { body } = await supertest + .post(`/api/transform/stop_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(isStopTransformsResponseSchema(body)).to.eql(true); + expect(body.invalid_transform_id.success).to.eql(false); + expect(body.invalid_transform_id).to.have.property('error'); + }); + }); + + describe('bulk stop', function () { + const reqBody: StopTransformsRequestSchema = [ + { id: 'bulk_stop_test_1', state: TRANSFORM_STATE.STARTED }, + { id: 'bulk_stop_test_2', state: TRANSFORM_STATE.STARTED }, + ]; + const destinationIndices = reqBody.map((d) => generateDestIndex(d.id)); + + beforeEach(async () => { + await asyncForEach(reqBody, async ({ id }: { id: string }, idx: number) => { + await createAndRunTransform(id); + }); + }); + + afterEach(async () => { + await transform.api.cleanTransformIndices(); + await asyncForEach(destinationIndices, async (destinationIndex: string) => { + await transform.api.deleteIndices(destinationIndex); + }); + }); + + it('should stop multiple transforms by transformIds', async () => { + const { body } = await supertest + .post(`/api/transform/stop_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(reqBody) + .expect(200); + + expect(isStopTransformsResponseSchema(body)).to.eql(true); + + await asyncForEach(reqBody, async ({ id: transformId }: { id: string }, idx: number) => { + expect(body[transformId].success).to.eql(true); + await transform.api.waitForTransformState(transformId, TRANSFORM_STATE.STOPPED); + await transform.api.waitForIndicesToExist(destinationIndices[idx]); + }); + }); + + it('should stop multiple transforms by transformIds, even if one of the transformIds is invalid', async () => { + const invalidTransformId = 'invalid_transform_id'; + const { body } = await supertest + .post(`/api/transform/stop_transforms`) + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send([ + { id: reqBody[0].id, state: reqBody[0].state }, + { id: invalidTransformId, state: TRANSFORM_STATE.STOPPED }, + { id: reqBody[1].id, state: reqBody[1].state }, + ]) + .expect(200); + + expect(isStopTransformsResponseSchema(body)).to.eql(true); + + await asyncForEach(reqBody, async ({ id: transformId }: { id: string }, idx: number) => { + expect(body[transformId].success).to.eql(true); + await transform.api.waitForTransformState(transformId, TRANSFORM_STATE.STOPPED); + await transform.api.waitForIndicesToExist(destinationIndices[idx]); + }); + + expect(body[invalidTransformId].success).to.eql(false); + expect(body[invalidTransformId]).to.have.property('error'); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/transform/transforms.ts b/x-pack/test/api_integration/apis/transform/transforms.ts new file mode 100644 index 0000000000000..c44c2b58e6207 --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/transforms.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import type { GetTransformsResponseSchema } from '../../../../plugins/transform/common/api_schemas/transforms'; +import { isGetTransformsResponseSchema } from '../../../../plugins/transform/common/api_schemas/type_guards'; +import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; +import { USER } from '../../../functional/services/transform/security_common'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +import { generateTransformConfig } from './common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const transform = getService('transform'); + + const expected = { + apiTransformTransforms: { + count: 2, + transform1: { id: 'transform-test-get-1', destIndex: 'user-transform-test-get-1' }, + transform2: { id: 'transform-test-get-2', destIndex: 'user-transform-test-get-2' }, + typeOfVersion: 'string', + typeOfCreateTime: 'number', + }, + apiTransformTransformsTransformId: { + count: 1, + transform1: { id: 'transform-test-get-1', destIndex: 'user-transform-test-get-1' }, + typeOfVersion: 'string', + typeOfCreateTime: 'number', + }, + }; + + async function createTransform(transformId: string) { + const config = generateTransformConfig(transformId); + await transform.api.createTransform(transformId, config); + } + + function assertTransformsResponseBody(body: GetTransformsResponseSchema) { + expect(isGetTransformsResponseSchema(body)).to.eql(true); + + expect(body.count).to.eql(expected.apiTransformTransforms.count); + expect(body.transforms).to.have.length(expected.apiTransformTransforms.count); + + const transform1 = body.transforms[0]; + expect(transform1.id).to.eql(expected.apiTransformTransforms.transform1.id); + expect(transform1.dest.index).to.eql(expected.apiTransformTransforms.transform1.destIndex); + expect(typeof transform1.version).to.eql(expected.apiTransformTransforms.typeOfVersion); + expect(typeof transform1.create_time).to.eql(expected.apiTransformTransforms.typeOfCreateTime); + + const transform2 = body.transforms[1]; + expect(transform2.id).to.eql(expected.apiTransformTransforms.transform2.id); + expect(transform2.dest.index).to.eql(expected.apiTransformTransforms.transform2.destIndex); + expect(typeof transform2.version).to.eql(expected.apiTransformTransforms.typeOfVersion); + expect(typeof transform2.create_time).to.eql(expected.apiTransformTransforms.typeOfCreateTime); + } + + function assertSingleTransformResponseBody(body: GetTransformsResponseSchema) { + expect(isGetTransformsResponseSchema(body)).to.eql(true); + + expect(body.count).to.eql(expected.apiTransformTransformsTransformId.count); + expect(body.transforms).to.have.length(expected.apiTransformTransformsTransformId.count); + + const transform1 = body.transforms[0]; + expect(transform1.id).to.eql(expected.apiTransformTransformsTransformId.transform1.id); + expect(transform1.dest.index).to.eql( + expected.apiTransformTransformsTransformId.transform1.destIndex + ); + expect(typeof transform1.version).to.eql( + expected.apiTransformTransformsTransformId.typeOfVersion + ); + expect(typeof transform1.create_time).to.eql( + expected.apiTransformTransformsTransformId.typeOfCreateTime + ); + } + + describe('/api/transform/transforms', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await transform.testResources.setKibanaTimeZoneToUTC(); + await createTransform('transform-test-get-1'); + await createTransform('transform-test-get-2'); + }); + + after(async () => { + await transform.api.cleanTransformIndices(); + }); + + describe('/transforms', function () { + it('should return a list of transforms for super-user', async () => { + const { body } = await supertest + .get('/api/transform/transforms') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + assertTransformsResponseBody(body); + }); + + it('should return a list of transforms for transform view-only user', async () => { + const { body } = await supertest + .get(`/api/transform/transforms`) + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + assertTransformsResponseBody(body); + }); + }); + + describe('/transforms/{transformId}', function () { + it('should return a specific transform configuration for super-user', async () => { + const { body } = await supertest + .get('/api/transform/transforms/transform-test-get-1') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + assertSingleTransformResponseBody(body); + }); + + it('should return a specific transform configuration transform view-only user', async () => { + const { body } = await supertest + .get(`/api/transform/transforms/transform-test-get-1`) + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + assertSingleTransformResponseBody(body); + }); + + it('should report 404 for a non-existing transform', async () => { + await supertest + .get('/api/transform/transforms/the-non-existing-transform') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(404); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/transform/transforms_preview.ts b/x-pack/test/api_integration/apis/transform/transforms_preview.ts new file mode 100644 index 0000000000000..d0fc44cf28fdb --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/transforms_preview.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import type { PostTransformsPreviewRequestSchema } from '../../../../plugins/transform/common/api_schemas/transforms'; + +import { FtrProviderContext } from '../../ftr_provider_context'; +import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; +import { USER } from '../../../functional/services/transform/security_common'; + +import { generateTransformConfig } from './common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const transform = getService('transform'); + + const expected = { + apiTransformTransformsPreview: { + previewItemCount: 19, + typeOfGeneratedDestIndex: 'object', + }, + }; + + function getTransformPreviewConfig() { + // passing in an empty string for transform id since we will not use + // it as part of the config request schema. Destructuring will + // remove the `dest` part of the config. + const { dest, ...config } = generateTransformConfig(''); + return config as PostTransformsPreviewRequestSchema; + } + + describe('/api/transform/transforms/_preview', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await transform.testResources.setKibanaTimeZoneToUTC(); + await transform.api.waitForIndicesToExist('ft_farequote'); + }); + + it('should return a transform preview', async () => { + const { body } = await supertest + .post('/api/transform/transforms/_preview') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(getTransformPreviewConfig()) + .expect(200); + + expect(body.preview).to.have.length(expected.apiTransformTransformsPreview.previewItemCount); + expect(typeof body.generated_dest_index).to.eql( + expected.apiTransformTransformsPreview.typeOfGeneratedDestIndex + ); + }); + + it('should return 403 for transform view-only user', async () => { + await supertest + .post(`/api/transform/transforms/_preview`) + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(getTransformPreviewConfig()) + .expect(403); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/transform/transforms_stats.ts b/x-pack/test/api_integration/apis/transform/transforms_stats.ts new file mode 100644 index 0000000000000..07856e5095a98 --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/transforms_stats.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import type { GetTransformsStatsResponseSchema } from '../../../../plugins/transform/common/api_schemas/transforms_stats'; +import { isGetTransformsStatsResponseSchema } from '../../../../plugins/transform/common/api_schemas/type_guards'; +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + +import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; +import { USER } from '../../../functional/services/transform/security_common'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +import { generateTransformConfig } from './common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const transform = getService('transform'); + + const expected = { + apiTransformTransforms: { + count: 2, + transform1: { id: 'transform-test-stats-1', state: TRANSFORM_STATE.STOPPED }, + transform2: { id: 'transform-test-stats-2', state: TRANSFORM_STATE.STOPPED }, + typeOfStats: 'object', + typeOfCheckpointing: 'object', + }, + }; + + async function createTransform(transformId: string) { + const config = generateTransformConfig(transformId); + await transform.api.createTransform(transformId, config); + } + + function assertTransformsStatsResponseBody(body: GetTransformsStatsResponseSchema) { + expect(isGetTransformsStatsResponseSchema(body)).to.eql(true); + expect(body.count).to.eql(expected.apiTransformTransforms.count); + expect(body.transforms).to.have.length(expected.apiTransformTransforms.count); + + const transform1 = body.transforms[0]; + expect(transform1.id).to.eql(expected.apiTransformTransforms.transform1.id); + expect(transform1.state).to.eql(expected.apiTransformTransforms.transform1.state); + expect(typeof transform1.stats).to.eql(expected.apiTransformTransforms.typeOfStats); + expect(typeof transform1.checkpointing).to.eql( + expected.apiTransformTransforms.typeOfCheckpointing + ); + + const transform2 = body.transforms[1]; + expect(transform2.id).to.eql(expected.apiTransformTransforms.transform2.id); + expect(transform2.state).to.eql(expected.apiTransformTransforms.transform2.state); + expect(typeof transform2.stats).to.eql(expected.apiTransformTransforms.typeOfStats); + expect(typeof transform2.checkpointing).to.eql( + expected.apiTransformTransforms.typeOfCheckpointing + ); + } + + describe('/api/transform/transforms/_stats', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await transform.testResources.setKibanaTimeZoneToUTC(); + await createTransform('transform-test-stats-1'); + await createTransform('transform-test-stats-2'); + }); + + after(async () => { + await transform.api.cleanTransformIndices(); + }); + + it('should return a list of transforms statistics for super-user', async () => { + const { body } = await supertest + .get('/api/transform/transforms/_stats') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + assertTransformsStatsResponseBody(body); + }); + + it('should return a list of transforms statistics view-only user', async () => { + const { body } = await supertest + .get(`/api/transform/transforms/_stats`) + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + assertTransformsStatsResponseBody(body); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/transform/transforms_update.ts b/x-pack/test/api_integration/apis/transform/transforms_update.ts new file mode 100644 index 0000000000000..3ad5b5b47c79b --- /dev/null +++ b/x-pack/test/api_integration/apis/transform/transforms_update.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api'; +import { USER } from '../../../functional/services/transform/security_common'; + +import { generateTransformConfig } from './common'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const transform = getService('transform'); + + const expected = { + transformOriginalConfig: { + count: 1, + id: 'transform-test-update-1', + source: { + index: ['ft_farequote'], + query: { match_all: {} }, + }, + }, + apiTransformTransformsPreview: { + previewItemCount: 19, + typeOfGeneratedDestIndex: 'object', + }, + }; + + async function createTransform(transformId: string) { + const config = generateTransformConfig(transformId); + await transform.api.createTransform(transformId, config); + } + + function getTransformUpdateConfig() { + return { + source: { + index: 'ft_*', + query: { + term: { + airline: { + value: 'AAL', + }, + }, + }, + }, + description: 'the-updated-description', + dest: { + index: 'user-the-updated-destination-index', + }, + frequency: '60m', + }; + } + + describe('/api/transform/transforms/{transformId}/_update', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await transform.testResources.setKibanaTimeZoneToUTC(); + await createTransform('transform-test-update-1'); + }); + + after(async () => { + await transform.api.cleanTransformIndices(); + }); + + it('should update a transform', async () => { + // assert the original transform for comparison + const { body: transformOriginalBody } = await supertest + .get('/api/transform/transforms/transform-test-update-1') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + expect(transformOriginalBody.count).to.eql(expected.transformOriginalConfig.count); + expect(transformOriginalBody.transforms).to.have.length( + expected.transformOriginalConfig.count + ); + + const transformOriginalConfig = transformOriginalBody.transforms[0]; + expect(transformOriginalConfig.id).to.eql(expected.transformOriginalConfig.id); + expect(transformOriginalConfig.source).to.eql(expected.transformOriginalConfig.source); + expect(transformOriginalConfig.description).to.eql(undefined); + expect(transformOriginalConfig.settings).to.eql({}); + + // update the transform and assert the response + const { body: transformUpdateResponseBody } = await supertest + .post('/api/transform/transforms/transform-test-update-1/_update') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(getTransformUpdateConfig()) + .expect(200); + + const expectedUpdateConfig = getTransformUpdateConfig(); + expect(transformUpdateResponseBody.id).to.eql(expected.transformOriginalConfig.id); + expect(transformUpdateResponseBody.source).to.eql({ + ...expectedUpdateConfig.source, + index: ['ft_*'], + }); + expect(transformUpdateResponseBody.description).to.eql(expectedUpdateConfig.description); + expect(transformUpdateResponseBody.settings).to.eql({}); + + // assert the updated transform for comparison + const { body: transformUpdatedBody } = await supertest + .get('/api/transform/transforms/transform-test-update-1') + .auth( + USER.TRANSFORM_POWERUSER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER) + ) + .set(COMMON_REQUEST_HEADERS) + .send() + .expect(200); + + expect(transformUpdatedBody.count).to.eql(expected.transformOriginalConfig.count); + expect(transformUpdatedBody.transforms).to.have.length( + expected.transformOriginalConfig.count + ); + + const transformUpdatedConfig = transformUpdatedBody.transforms[0]; + expect(transformUpdatedConfig.id).to.eql(expected.transformOriginalConfig.id); + expect(transformUpdatedConfig.source).to.eql({ + ...expectedUpdateConfig.source, + index: ['ft_*'], + }); + expect(transformUpdatedConfig.description).to.eql(expectedUpdateConfig.description); + expect(transformUpdatedConfig.settings).to.eql({}); + }); + + it('should return 403 for transform view-only user', async () => { + await supertest + .post('/api/transform/transforms/transform-test-update-1/_update') + .auth( + USER.TRANSFORM_VIEWER, + transform.securityCommon.getPasswordForUser(USER.TRANSFORM_VIEWER) + ) + .set(COMMON_REQUEST_HEADERS) + .send(getTransformUpdateConfig()) + .expect(403); + }); + }); +}; diff --git a/x-pack/test/functional/apps/transform/cloning.ts b/x-pack/test/functional/apps/transform/cloning.ts index b6ccd68bb2096..a147b56d56251 100644 --- a/x-pack/test/functional/apps/transform/cloning.ts +++ b/x-pack/test/functional/apps/transform/cloning.ts @@ -5,12 +5,12 @@ */ import { FtrProviderContext } from '../../ftr_provider_context'; -import { TransformPivotConfig } from '../../../../plugins/transform/public/app/common'; +import { TransformPivotConfig } from '../../../../plugins/transform/common/types/transform'; function getTransformConfig(): TransformPivotConfig { const date = Date.now(); return { - id: `ec_2_${date}`, + id: `ec_cloning_${date}`, source: { index: ['ft_ecommerce'] }, pivot: { group_by: { category: { terms: { field: 'category.keyword' } } }, @@ -32,7 +32,7 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { await esArchiver.loadIfNeeded('ml/ecommerce'); await transform.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date'); - await transform.api.createAndRunTransform(transformConfig); + await transform.api.createAndRunTransform(transformConfig.id, transformConfig); await transform.testResources.setKibanaTimeZoneToUTC(); await transform.securityUI.loginAsTransformPowerUser(); diff --git a/x-pack/test/functional/apps/transform/creation_index_pattern.ts b/x-pack/test/functional/apps/transform/creation_index_pattern.ts index 4e2b832838b7d..13213679a6117 100644 --- a/x-pack/test/functional/apps/transform/creation_index_pattern.ts +++ b/x-pack/test/functional/apps/transform/creation_index_pattern.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + import { FtrProviderContext } from '../../ftr_provider_context'; interface GroupByEntry { @@ -141,7 +143,7 @@ export default function ({ getService }: FtrProviderContext) { values: [`Men's Accessories`], }, row: { - status: 'stopped', + status: TRANSFORM_STATE.STOPPED, mode: 'batch', progress: '100', }, @@ -239,7 +241,7 @@ export default function ({ getService }: FtrProviderContext) { values: ['AE', 'CO', 'EG', 'FR', 'GB'], }, row: { - status: 'stopped', + status: TRANSFORM_STATE.STOPPED, mode: 'batch', progress: '100', }, diff --git a/x-pack/test/functional/apps/transform/creation_saved_search.ts b/x-pack/test/functional/apps/transform/creation_saved_search.ts index 229ff97782362..20d276c2e017b 100644 --- a/x-pack/test/functional/apps/transform/creation_saved_search.ts +++ b/x-pack/test/functional/apps/transform/creation_saved_search.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + import { FtrProviderContext } from '../../ftr_provider_context'; interface GroupByEntry { @@ -58,7 +60,7 @@ export default function ({ getService }: FtrProviderContext) { values: ['ASA'], }, row: { - status: 'stopped', + status: TRANSFORM_STATE.STOPPED, mode: 'batch', progress: '100', }, diff --git a/x-pack/test/functional/apps/transform/editing.ts b/x-pack/test/functional/apps/transform/editing.ts index 460e7c5b24a98..ac955bde4ad5d 100644 --- a/x-pack/test/functional/apps/transform/editing.ts +++ b/x-pack/test/functional/apps/transform/editing.ts @@ -4,13 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +import { TransformPivotConfig } from '../../../../plugins/transform/common/types/transform'; +import { TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; + import { FtrProviderContext } from '../../ftr_provider_context'; -import { TransformPivotConfig } from '../../../../plugins/transform/public/app/common'; function getTransformConfig(): TransformPivotConfig { const date = Date.now(); return { - id: `ec_2_${date}`, + id: `ec_editing_${date}`, source: { index: ['ft_ecommerce'] }, pivot: { group_by: { category: { terms: { field: 'category.keyword' } } }, @@ -32,7 +34,7 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { await esArchiver.loadIfNeeded('ml/ecommerce'); await transform.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date'); - await transform.api.createAndRunTransform(transformConfig); + await transform.api.createAndRunTransform(transformConfig.id, transformConfig); await transform.testResources.setKibanaTimeZoneToUTC(); await transform.securityUI.loginAsTransformPowerUser(); @@ -52,7 +54,7 @@ export default function ({ getService }: FtrProviderContext) { expected: { messageText: 'updated transform.', row: { - status: 'stopped', + status: TRANSFORM_STATE.STOPPED, mode: 'batch', progress: '100', }, diff --git a/x-pack/test/functional/services/transform/api.ts b/x-pack/test/functional/services/transform/api.ts index 697020fafb196..d97db93c31b3b 100644 --- a/x-pack/test/functional/services/transform/api.ts +++ b/x-pack/test/functional/services/transform/api.ts @@ -5,13 +5,17 @@ */ import expect from '@kbn/expect'; +import type { PutTransformsRequestSchema } from '../../../../plugins/transform/common/api_schemas/transforms'; +import { TransformState, TRANSFORM_STATE } from '../../../../plugins/transform/common/constants'; +import type { TransformStats } from '../../../../plugins/transform/common/types/transform_stats'; + import { FtrProviderContext } from '../../ftr_provider_context'; -import { TRANSFORM_STATE } from '../../../../plugins/transform/common'; -import { - TransformPivotConfig, - TransformStats, -} from '../../../../plugins/transform/public/app/common'; +export async function asyncForEach(array: any[], callback: Function) { + for (let index = 0; index < array.length; index++) { + await callback(array[index], index, array); + } +} export function TransformAPIProvider({ getService }: FtrProviderContext) { const es = getService('legacyEs'); @@ -35,7 +39,7 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { await this.waitForIndicesToExist(indices, `expected ${indices} to be created`); }, - async deleteIndices(indices: string) { + async deleteIndices(indices: string, skipWaitForIndicesNotToExist?: boolean) { log.debug(`Deleting indices: '${indices}'...`); if ((await es.indices.exists({ index: indices, allowNoIndices: false })) === false) { log.debug(`Indices '${indices}' don't exist. Nothing to delete.`); @@ -49,7 +53,13 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { .to.have.property('acknowledged') .eql(true, 'Response for delete request should be acknowledged'); - await this.waitForIndicesNotToExist(indices, `expected indices '${indices}' to be deleted`); + // Check for the option to skip the check if the indices are deleted. + // For example, we might want to clear the .transform-* indices but they + // will be automatically regenerated making tests flaky without the option + // to skip this check. + if (!skipWaitForIndicesNotToExist) { + await this.waitForIndicesNotToExist(indices, `expected indices '${indices}' to be deleted`); + } }, async waitForIndicesToExist(indices: string, errorMsg?: string) { @@ -73,7 +83,26 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { }, async cleanTransformIndices() { - await this.deleteIndices('.transform-*'); + // Delete all transforms using the API since we mustn't just delete + // all `.transform-*` indices since this might result in orphaned ES tasks. + const { + body: { transforms }, + } = await esSupertest.get(`/_transform/`).expect(200); + const transformIds = transforms.map((t: { id: string }) => t.id); + + await asyncForEach(transformIds, async (transformId: string) => { + await esSupertest + .post(`/_transform/${transformId}/_stop?force=true&wait_for_completion`) + .expect(200); + await this.waitForTransformState(transformId, TRANSFORM_STATE.STOPPED); + + await esSupertest.delete(`/_transform/${transformId}`).expect(200); + await this.waitForTransformNotToExist(transformId); + }); + + // Delete all transform related notifications to clear messages tabs + // in the transforms list expanded rows. + await this.deleteIndices('.transform-notifications-*'); }, async getTransformStats(transformId: string): Promise { @@ -90,12 +119,12 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { return statsResponse.transforms[0]; }, - async getTransformState(transformId: string): Promise { + async getTransformState(transformId: string): Promise { const stats = await this.getTransformStats(transformId); return stats.state; }, - async waitForTransformState(transformId: string, expectedState: TRANSFORM_STATE) { + async waitForTransformState(transformId: string, expectedState: TransformState) { await retry.waitForWithTimeout( `transform state to be ${expectedState}`, 2 * 60 * 1000, @@ -110,6 +139,23 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { ); }, + async waitForTransformStateNotToBe(transformId: string, notExpectedState: TransformState) { + await retry.waitForWithTimeout( + `transform state not to be ${notExpectedState}`, + 2 * 60 * 1000, + async () => { + const state = await this.getTransformState(transformId); + if (state !== notExpectedState) { + return true; + } else { + throw new Error( + `expected transform state to not be ${notExpectedState} but got ${state}` + ); + } + } + ); + }, + async waitForBatchTransformToComplete(transformId: string) { await retry.waitForWithTimeout(`batch transform to complete`, 2 * 60 * 1000, async () => { const stats = await this.getTransformStats(transformId); @@ -127,8 +173,7 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { return await esSupertest.get(`/_transform/${transformId}`).expect(expectedCode); }, - async createTransform(transformConfig: TransformPivotConfig) { - const transformId = transformConfig.id; + async createTransform(transformId: string, transformConfig: PutTransformsRequestSchema) { log.debug(`Creating transform with id '${transformId}'...`); await esSupertest.put(`/_transform/${transformId}`).send(transformConfig).expect(200); @@ -147,6 +192,7 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { } }); }, + async waitForTransformNotToExist(transformId: string, errorMsg?: string) { await retry.waitForWithTimeout(`'${transformId}' to exist`, 5 * 1000, async () => { if (await this.getTransform(transformId, 404)) { @@ -162,15 +208,15 @@ export function TransformAPIProvider({ getService }: FtrProviderContext) { await esSupertest.post(`/_transform/${transformId}/_start`).expect(200); }, - async createAndRunTransform(transformConfig: TransformPivotConfig) { - await this.createTransform(transformConfig); - await this.startTransform(transformConfig.id); + async createAndRunTransform(transformId: string, transformConfig: PutTransformsRequestSchema) { + await this.createTransform(transformId, transformConfig); + await this.startTransform(transformId); if (transformConfig.sync === undefined) { // batch mode - await this.waitForBatchTransformToComplete(transformConfig.id); + await this.waitForBatchTransformToComplete(transformId); } else { // continuous mode - await this.waitForTransformState(transformConfig.id, TRANSFORM_STATE.STARTED); + await this.waitForTransformStateNotToBe(transformId, TRANSFORM_STATE.STOPPED); } }, }; diff --git a/x-pack/test/functional/services/transform/transform_table.ts b/x-pack/test/functional/services/transform/transform_table.ts index 77e52b642261b..cc360379f32c3 100644 --- a/x-pack/test/functional/services/transform/transform_table.ts +++ b/x-pack/test/functional/services/transform/transform_table.ts @@ -174,7 +174,7 @@ export function TransformTableProvider({ getService }: FtrProviderContext) { await testSubjects.existOrFail('transformMessagesTab'); await testSubjects.click('transformMessagesTab'); await testSubjects.existOrFail('~transformMessagesTabContent'); - await retry.tryForTime(5000, async () => { + await retry.tryForTime(30 * 1000, async () => { const actualText = await testSubjects.getVisibleText('~transformMessagesTabContent'); expect(actualText.includes(expectedText)).to.eql( true, From 8eb4b2e471dafb8fdde98376d34a73cce125026d Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Mon, 14 Sep 2020 17:39:11 +0300 Subject: [PATCH 3/5] [telemetry] add schema guideline + schema_check new check for --path config (#75747) --- packages/kbn-telemetry-tools/GUIDELINE.md | 211 ++++++++++++++++++ .../src/cli/run_telemetry_check.ts | 27 ++- 2 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 packages/kbn-telemetry-tools/GUIDELINE.md diff --git a/packages/kbn-telemetry-tools/GUIDELINE.md b/packages/kbn-telemetry-tools/GUIDELINE.md new file mode 100644 index 0000000000000..e7d09babbf9e2 --- /dev/null +++ b/packages/kbn-telemetry-tools/GUIDELINE.md @@ -0,0 +1,211 @@ +# Collector Schema Guideline + +Table of contents: +- [Collector Schema Guideline](#collector-schema-guideline) + - [Adding schema to your collector](#adding-schema-to-your-collector) + - [1. Update the telemetryrc file](#1-update-the-telemetryrc-file) + - [2. Type the `fetch` function](#2-type-the-fetch-function) + - [3. Add a `schema` field](#3-add-a-schema-field) + - [4. Run the telemetry check](#4-run-the-telemetry-check) + - [5. Update the stored json files](#5-update-the-stored-json-files) + - [Updating the collector schema](#updating-the-collector-schema) + - [Writing the schema](#writing-the-schema) + - [Basics](#basics) + - [Allowed types](#allowed-types) + - [Dealing with arrays](#dealing-with-arrays) + - [Schema Restrictions](#schema-restrictions) + - [Root of schema can only be an object](#root-of-schema-can-only-be-an-object) + + +## Adding schema to your collector + +To add a `schema` to the collector, follow these steps until the telemetry check passes. +To check the next step needed simply run the telemetry check with the path of your collector: + +``` +node scripts/telemetry_check.js --path=.ts +``` + +### 1. Update the telemetryrc file + +Make sure your collector is not excluded in the `telemetryrc.json` files (located at the root of the kibana project, and another on in the `x-pack` dir). + +```s +[ + { + ... + "exclude": [ + "" + ] + } +] +``` + +Note that the check will fail if the collector in --path is excluded. + +### 2. Type the `fetch` function +1. Make sure the return of the `fetch` function is typed. + +The function `makeUsageCollector` accepts a generic type parameter of the returned type of the `fetch` function. + +``` +interface Usage { + someStat: number; +} + +usageCollection.makeUsageCollector({ + fetch: async () => { + return { + someStat: 3, + } + }, + ... +}) +``` + +The generic type passed to `makeUsageCollector` will automatically unwrap the `Promise` to check for the resolved type. + +### 3. Add a `schema` field + +Add a `schema` field to your collector. After passing the return type of the fetch function to the `makeUsageCollector` generic parameter. It will automaticallly figure out the correct type of the schema based on that provided type. + + +``` +interface Usage { + someStat: number; +} + +usageCollection.makeUsageCollector({ + schema: { + someStat: { + type: 'long' + } + }, + ... +}) +``` + +For full details on writing the `schema` object, check the [Writing the schema](#writing-the-schema) section. + +### 4. Run the telemetry check + +To make sure your changes pass the telemetry check you can run the following: + +``` +node scripts/telemetry_check.js --ignore-stored-json --path=.ts +``` + +### 5. Update the stored json files + +The `--fix` flag will automatically update the persisted json files used by the telemetry team. + +``` +node scripts/telemetry_check.js --fix +``` + +Note that any updates to the stored json files will require a review by the kibana-telemetry team to help us update the telemetry cluster mappings and ensure your changes adhere to our best practices. + + +## Updating the collector schema + +Simply update the fetch function to start returning the updated fields back to our cluster. The update the schema to accomodate these changes. + +Once youre run the changes to both the `fetch` function and the `schema` field run the following command + +``` +node scripts/telemetry_check.js --fix +``` + +The `--fix` flag will automatically update the persisted json files used by the telemetry team. Note that any updates to the stored json files will require a review by the kibana-telemetry team to help us update the telemetry cluster mappings and ensure your changes adhere to our best practices. + + +## Writing the schema + +We've designed the schema object to closely resemble elasticsearch mapping object to reduce any cognitive complexity. + +### Basics + +The function `makeUsageCollector` will automatically translate the returned `Usage` fetch type to the `schema` object. This way you'll have the typescript type checker helping you write the correct corrisponding schema. + +``` +interface Usage { + someStat: number; +} + +usageCollection.makeUsageCollector({ + schema: { + someStat: { + type: 'long' + } + }, + ... +}) +``` + + +### Allowed types + +Any field property in the schema accepts a `type` field. By default the type is `object` which accepts nested properties under it. Currently we accept the following property types: + +``` +AllowedSchemaTypes = + | 'keyword' + | 'text' + | 'number' + | 'boolean' + | 'long' + | 'date' + | 'float'; +``` + + +### Dealing with arrays + +You can optionally define a property to be an array by setting the `isArray` to `true`. Note that the `isArray` property is not currently required. + + +``` +interface Usage { + arrayOfStrings: string[]; + arrayOfObjects: {key: string; value: number; }[]; +} + +usageCollection.makeUsageCollector({ + fetch: () => { + return { + arrayOfStrings: ['item_one', 'item_two'], + arrayOfObjects: [ + { key: 'key_one', value: 13 }, + ] + } + } + schema: { + arrayOfStrings: { + type: 'keyword', + isArray: true, + }, + arrayOfObjects: { + isArray: true, + key: { + type: 'keyword', + }, + value: { + type: 'long', + }, + } + }, + ... +}) +``` + +Be careful adding arrays of objects due to the limitation in correlating the properties inside those objects inside kibana. It is advised to look for an alternative schema based on your use cases. + + +## Schema Restrictions + +We have enforced some restrictions to the schema object to adhere to our telemetry best practices. These practices are derived from the usablity of the sent data in our telemetry cluster. + + +### Root of schema can only be an object + +The root of the schema can only be an object. Currently any property must be nested inside the main schema object. \ No newline at end of file diff --git a/packages/kbn-telemetry-tools/src/cli/run_telemetry_check.ts b/packages/kbn-telemetry-tools/src/cli/run_telemetry_check.ts index 2f85fd2cdd2a4..87ba68c1bcb27 100644 --- a/packages/kbn-telemetry-tools/src/cli/run_telemetry_check.ts +++ b/packages/kbn-telemetry-tools/src/cli/run_telemetry_check.ts @@ -35,7 +35,7 @@ import { export function runTelemetryCheck() { run( - async ({ flags: { fix = false, path }, log }) => { + async ({ flags: { fix = false, 'ignore-stored-json': ignoreStoredJson, path }, log }) => { if (typeof fix !== 'boolean') { throw createFailError(`${chalk.white.bgRed(' TELEMETRY ERROR ')} --fix can't have a value`); } @@ -50,6 +50,14 @@ export function runTelemetryCheck() { ); } + if (fix && typeof ignoreStoredJson !== 'undefined') { + throw createFailError( + `${chalk.white.bgRed( + ' TELEMETRY ERROR ' + )} --fix is incompatible with --ignore-stored-json flag.` + ); + } + const list = new Listr([ { title: 'Checking .telemetryrc.json files', @@ -59,11 +67,28 @@ export function runTelemetryCheck() { title: 'Extracting Collectors', task: (context) => new Listr(extractCollectorsTask(context, path), { exitOnError: true }), }, + { + enabled: () => typeof path !== 'undefined', + title: 'Checking collectors in --path are not excluded', + task: ({ roots }: TaskContext) => { + const totalCollections = roots.reduce((acc, root) => { + return acc + (root.parsedCollections?.length || 0); + }, 0); + const collectorsInPath = Array.isArray(path) ? path.length : 1; + + if (totalCollections !== collectorsInPath) { + throw new Error( + 'Collector specified in `path` is excluded; Check the telemetryrc.json files.' + ); + } + }, + }, { title: 'Checking Compatible collector.schema with collector.fetch type', task: (context) => new Listr(checkCompatibleTypesTask(context), { exitOnError: true }), }, { + enabled: (_) => !!ignoreStoredJson, title: 'Checking Matching collector.schema against stored json files', task: (context) => new Listr(checkMatchingSchemasTask(context, !fix), { exitOnError: true }), From c5358f3a9b034b93899bdbf988a0bac5212b6d6d Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 14 Sep 2020 16:59:09 +0200 Subject: [PATCH 4/5] [ML] Fix custom URLs processing for security app (#76957) * [ML] fix custom urls processing for security app * [ML] improve query string parsing * [ML] remove escaping with !, adjust a unit test for security app * [ML] unit test * [ML] unit test --- .../application/util/custom_url_utils.test.ts | 109 +++++++++++++- .../application/util/custom_url_utils.ts | 137 ++++++++++-------- 2 files changed, 186 insertions(+), 60 deletions(-) diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts index 2912aad6819cf..6a5583ecbb8ac 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -61,8 +61,13 @@ describe('ML - custom URL utils', () => { influencer_field_name: 'airline', influencer_field_values: ['<>:;[}")'], }, + { + influencer_field_name: 'odd:field,name', + influencer_field_values: [">:&12<'"], + }, ], airline: ['<>:;[}")'], + 'odd:field,name': [">:&12<'"], }; const TEST_RECORD_MULTIPLE_INFLUENCER_VALUES: CustomUrlAnomalyRecordDoc = { @@ -98,7 +103,7 @@ describe('ML - custom URL utils', () => { url_name: 'Raw data', time_range: 'auto', url_value: - "discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"$airline$\"'))", + "discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"$airline$\" and odd:field,name : $odd:field,name$'))", }; const TEST_DASHBOARD_LUCENE_URL: KibanaUrlConfig = { @@ -263,9 +268,55 @@ describe('ML - custom URL utils', () => { ); }); - test('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => { + test.skip('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => { expect(getUrlForRecord(TEST_DISCOVER_URL, TEST_RECORD_SPECIAL_CHARS)).toBe( - "discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\"'))" + "discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\" and odd:field,name:>:&12<''))" + ); + }); + + test('correctly encodes special characters inside of a query string', () => { + const testUrl = { + url_name: 'Show dashboard', + time_range: 'auto', + url_value: `dashboards#/view/351de820-f2bb-11ea-ab06-cb93221707e9?_a=(filters:!(),query:(language:kuery,query:'at@name:"$at@name$" and singlequote!'name:"$singlequote!'name$"'))&_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))`, + }; + + const testRecord = { + job_id: 'spec-char', + result_type: 'record', + probability: 0.0028099428534745633, + multi_bucket_impact: 5, + record_score: 49.00785814424704, + initial_record_score: 49.00785814424704, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1549593000000, + partition_field_name: 'at@name', + partition_field_value: "contains a ' quote", + function: 'mean', + function_description: 'mean', + typical: [1993.2657340111837], + actual: [1808.3334418402778], + field_name: 'metric%$£&!{(]field', + influencers: [ + { + influencer_field_name: "singlequote'name", + influencer_field_values: ["contains a ' quote"], + }, + { + influencer_field_name: 'at@name', + influencer_field_values: ["contains a ' quote"], + }, + ], + "singlequote'name": ["contains a ' quote"], + 'at@name': ["contains a ' quote"], + earliest: '2019-02-08T00:00:00.000Z', + latest: '2019-02-08T23:59:59.999Z', + }; + + expect(getUrlForRecord(testUrl, testRecord)).toBe( + `dashboards#/view/351de820-f2bb-11ea-ab06-cb93221707e9?_a=(filters:!(),query:(language:kuery,query:'at@name:"contains%20a%20!'%20quote" AND singlequote!'name:"contains%20a%20!'%20quote"'))&_g=(filters:!(),time:(from:'2019-02-08T00:00:00.000Z',mode:absolute,to:'2019-02-08T23:59:59.999Z'))` ); }); @@ -405,6 +456,58 @@ describe('ML - custom URL utils', () => { ); }); + test('return expected url for Security app', () => { + const urlConfig = { + url_name: 'Hosts Details by process name', + url_value: + "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))", + }; + + const testRecords = { + job_id: 'rare_process_by_host_linux_ecs', + result_type: 'record', + probability: 0.018122957282324745, + multi_bucket_impact: 0, + record_score: 20.513469583273547, + initial_record_score: 20.513469583273547, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1549043100000, + by_field_name: 'process.name', + by_field_value: 'seq', + partition_field_name: 'host.name', + partition_field_value: 'showcase', + function: 'rare', + function_description: 'rare', + typical: [0.018122957282324745], + actual: [1], + influencers: [ + { + influencer_field_name: 'user.name', + influencer_field_values: ['sophie'], + }, + { + influencer_field_name: 'process.name', + influencer_field_values: ['seq'], + }, + { + influencer_field_name: 'host.name', + influencer_field_values: ['showcase'], + }, + ], + 'process.name': ['seq'], + 'user.name': ['sophie'], + 'host.name': ['showcase'], + earliest: '2019-02-01T16:00:00.000Z', + latest: '2019-02-01T18:59:59.999Z', + }; + + expect(getUrlForRecord(urlConfig, testRecords)).toBe( + "security/hosts/ml-hosts/showcase?_g=()&query=(language:kuery,query:'process.name:\"seq\"')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))" + ); + }); + test('removes an empty path component with a trailing slash', () => { const urlConfig = { url_name: 'APM', diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts index 8263def2034aa..18ba1e4ee337b 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts @@ -8,6 +8,7 @@ import { get, flow } from 'lodash'; import moment from 'moment'; +import rison, { RisonObject, RisonValue } from 'rison-node'; import { parseInterval } from '../../../common/util/parse_interval'; import { escapeForElasticsearchQuery, replaceStringTokens } from './string_utils'; @@ -131,13 +132,70 @@ function escapeForKQL(value: string | number): string { type GetResultTokenValue = (v: string) => string; +export const isRisonObject = (value: RisonValue): value is RisonObject => { + return value !== null && typeof value === 'object'; +}; + +const getQueryStringResultProvider = ( + record: CustomUrlAnomalyRecordDoc, + getResultTokenValue: GetResultTokenValue +) => (resultPrefix: string, queryString: string, resultPostfix: string): string => { + const URL_LENGTH_LIMIT = 2000; + + let availableCharactersLeft = URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length; + + // URL template might contain encoded characters + const queryFields = queryString + // Split query string by AND operator. + .split(/\sand\s/i) + // Get property name from `influencerField:$influencerField$` string. + .map((v) => String(v.split(/:(.+)?\$/)[0]).trim()); + + const queryParts: string[] = []; + const joinOperator = ' AND '; + + fieldsLoop: for (let i = 0; i < queryFields.length; i++) { + const field = queryFields[i]; + // Use lodash get to allow nested JSON fields to be retrieved. + let tokenValues: string[] | string | null = get(record, field) || null; + if (tokenValues === null) { + continue; + } + tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues]; + + // Create a pair `influencerField:value`. + // In cases where there are multiple influencer field values for an anomaly + // combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`. + let result = ''; + for (let j = 0; j < tokenValues.length; j++) { + const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue(tokenValues[j])}"`; + + // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query. + if (availableCharactersLeft < part.length) { + if (result.length > 0) { + queryParts.push(j > 0 ? `(${result})` : result); + } + break fieldsLoop; + } + + result += part; + + availableCharactersLeft -= result.length; + } + + if (result.length > 0) { + queryParts.push(tokenValues.length > 1 ? `(${result})` : result); + } + } + return queryParts.join(joinOperator); +}; + /** * Builds a Kibana dashboard or Discover URL from the supplied config, with any * dollar delimited tokens substituted from the supplied anomaly record. */ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) { const urlValue = urlConfig.url_value; - const URL_LENGTH_LIMIT = 2000; const isLuceneQueryLanguage = urlValue.includes('language:lucene'); @@ -145,11 +203,7 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) ? escapeForElasticsearchQuery : escapeForKQL; - const commonEscapeCallback = flow( - // Kibana URLs used rison encoding, so escape with ! any ! or ' characters - (v: string): string => v.replace(/[!']/g, '!$&'), - encodeURIComponent - ); + const commonEscapeCallback = flow(encodeURIComponent); const replaceSingleTokenValues = (str: string) => { const getResultTokenValue: GetResultTokenValue = flow( @@ -172,65 +226,34 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) return flow( (str: string) => str.replace('$earliest$', record.earliest).replace('$latest$', record.latest), // Process query string content of the URL + decodeURIComponent, (str: string) => { const getResultTokenValue: GetResultTokenValue = flow( queryLanguageEscapeCallback, commonEscapeCallback ); + + const getQueryStringResult = getQueryStringResultProvider(record, getResultTokenValue); + + const match = str.match(/(.+)(\(.*\blanguage:(?:lucene|kuery)\b.*?\))(.+)/); + + if (match !== null && match[2] !== undefined) { + const [, prefix, queryDef, postfix] = match; + + const q = rison.decode(queryDef); + + if (isRisonObject(q) && q.hasOwnProperty('query')) { + const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues); + const resultQuery = getQueryStringResult(resultPrefix, q.query as string, resultPostfix); + return `${resultPrefix}${rison.encode({ ...q, query: resultQuery })}${resultPostfix}`; + } + } + return str.replace( - /(.+query:'|.+&kuery=)([^']*)(['&].+)/, + /(.+&kuery=)(.*?)[^!](&.+)/, (fullMatch, prefix: string, queryString: string, postfix: string) => { const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues); - - let availableCharactersLeft = - URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length; - const queryFields = queryString - // Split query string by AND operator. - .split(/\sand\s/i) - // Get property name from `influencerField:$influencerField$` string. - .map((v) => v.split(':')[0]); - - const queryParts: string[] = []; - const joinOperator = ' AND '; - - fieldsLoop: for (let i = 0; i < queryFields.length; i++) { - const field = queryFields[i]; - // Use lodash get to allow nested JSON fields to be retrieved. - let tokenValues: string[] | string | null = get(record, field) || null; - if (tokenValues === null) { - continue; - } - tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues]; - - // Create a pair `influencerField:value`. - // In cases where there are multiple influencer field values for an anomaly - // combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`. - let result = ''; - for (let j = 0; j < tokenValues.length; j++) { - const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue( - tokenValues[j] - )}"`; - - // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query. - if (availableCharactersLeft < part.length) { - if (result.length > 0) { - queryParts.push(j > 0 ? `(${result})` : result); - } - break fieldsLoop; - } - - result += part; - - availableCharactersLeft -= result.length; - } - - if (result.length > 0) { - queryParts.push(tokenValues.length > 1 ? `(${result})` : result); - } - } - - const resultQuery = queryParts.join(joinOperator); - + const resultQuery = getQueryStringResult(resultPrefix, queryString, resultPostfix); return `${resultPrefix}${resultQuery}${resultPostfix}`; } ); From 0c3a8c5f4ef4735ba5a784c6e916b3289834682b Mon Sep 17 00:00:00 2001 From: Peter Pisljar Date: Mon, 14 Sep 2020 17:00:23 +0200 Subject: [PATCH 5/5] updating datatable type (#77320) --- .../expressions/common/expression_types/specs/boolean.ts | 1 - .../common/expression_types/specs/datatable.ts | 9 +++------ .../expressions/common/expression_types/specs/num.ts | 1 - .../expressions/common/expression_types/specs/number.ts | 1 - .../expressions/common/expression_types/specs/string.ts | 1 - 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/plugins/expressions/common/expression_types/specs/boolean.ts b/src/plugins/expressions/common/expression_types/specs/boolean.ts index adbdeafc34fd2..73b0b98eaaf06 100644 --- a/src/plugins/expressions/common/expression_types/specs/boolean.ts +++ b/src/plugins/expressions/common/expression_types/specs/boolean.ts @@ -41,7 +41,6 @@ export const boolean: ExpressionTypeDefinition<'boolean', boolean> = { }, datatable: (value): Datatable => ({ type: 'datatable', - meta: {}, columns: [{ id: 'value', name: 'value', meta: { type: name } }], rows: [{ value }], }), diff --git a/src/plugins/expressions/common/expression_types/specs/datatable.ts b/src/plugins/expressions/common/expression_types/specs/datatable.ts index dd3c653878de7..c201e99faeb03 100644 --- a/src/plugins/expressions/common/expression_types/specs/datatable.ts +++ b/src/plugins/expressions/common/expression_types/specs/datatable.ts @@ -52,7 +52,10 @@ export type DatatableRow = Record; export interface DatatableColumnMeta { type: DatatableColumnType; field?: string; + index?: string; params?: SerializableState; + source?: string; + sourceParams?: SerializableState; } /** * This type represents the shape of a column in a `Datatable`. @@ -63,17 +66,11 @@ export interface DatatableColumn { meta: DatatableColumnMeta; } -export interface DatatableMeta { - type?: string; - source?: string; -} - /** * A `Datatable` in Canvas is a unique structure that represents tabulated data. */ export interface Datatable { type: typeof name; - meta?: DatatableMeta; columns: DatatableColumn[]; rows: DatatableRow[]; } diff --git a/src/plugins/expressions/common/expression_types/specs/num.ts b/src/plugins/expressions/common/expression_types/specs/num.ts index 041747f39740b..d208a9dcf73c8 100644 --- a/src/plugins/expressions/common/expression_types/specs/num.ts +++ b/src/plugins/expressions/common/expression_types/specs/num.ts @@ -73,7 +73,6 @@ export const num: ExpressionTypeDefinition<'num', ExpressionValueNum> = { }, datatable: ({ value }): Datatable => ({ type: 'datatable', - meta: {}, columns: [{ id: 'value', name: 'value', meta: { type: 'number' } }], rows: [{ value }], }), diff --git a/src/plugins/expressions/common/expression_types/specs/number.ts b/src/plugins/expressions/common/expression_types/specs/number.ts index c5fdacf3408a1..c30d3fe943d42 100644 --- a/src/plugins/expressions/common/expression_types/specs/number.ts +++ b/src/plugins/expressions/common/expression_types/specs/number.ts @@ -55,7 +55,6 @@ export const number: ExpressionTypeDefinition = { }, datatable: (value): Datatable => ({ type: 'datatable', - meta: {}, columns: [{ id: 'value', name: 'value', meta: { type: 'number' } }], rows: [{ value }], }), diff --git a/src/plugins/expressions/common/expression_types/specs/string.ts b/src/plugins/expressions/common/expression_types/specs/string.ts index 3d52707279bfc..0869e21e455f7 100644 --- a/src/plugins/expressions/common/expression_types/specs/string.ts +++ b/src/plugins/expressions/common/expression_types/specs/string.ts @@ -40,7 +40,6 @@ export const string: ExpressionTypeDefinition = { }, datatable: (value): Datatable => ({ type: 'datatable', - meta: {}, columns: [{ id: 'value', name: 'value', meta: { type: 'string' } }], rows: [{ value }], }),