From ba76b505b1faa2e3ca69d8eb2d822843fb8a6440 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 24 Apr 2024 11:28:59 +0200 Subject: [PATCH 001/138] [Obs AI Assistant] Instructions & Claude improvements (#181058) When we send over a conversation to the LLM for completion, we include a system message. System messages are a way for the consumer (in this case, us as developers) to control the LLM's behavior. This system message was previously constructed by using a concept called `ContextDefinition` - originally this was a way to define a set of functions and behavior for a specific context, e.g. core functionality, APM-specific functionality, platform-specific functionality etc. However we never actually did anything with this, and much of its intended functionality is now captured with the screen context API. In https://github.com/elastic/kibana/issues/179736, we added user instructions, which are ways for the user to control the Assistant's behaviour, by appending to the system message we construct with the registered context definitions. With this PR, we are making several changes: - Remove the concept of concept definitions entirely - Replace it with `registerInstruction`, which allows the consumer to register pieces of text that will be included in the system message. - `registerInstruction` _also_ takes a callback. That callback receives the available function names for that specific chat request. For instance, when we reach the function call limit, the LLM will have no functions to call. This allows consumers to cater their instructions to this specific scenario, which somewhat limits the possibility of the LLM calling a function that it is not allowed to - Claude is especially prone to this (likely related to the fact we use simulated function calling). This leads to the following functional changes: - A system message is now constructed by combining the registered instructions (system-specific) with the knowledge base and request instructions (user-specific) - `GET /internal/observability_ai_assistant/functions` no longer returns the contexts. Instead it returns the system message - `GET /internal/observability_ai_assistant/chat/complete` now creates a system message at the start, and overrides the system message from the request. - For each invocation of `chat`, it re-calculates the system message by "materializing" the registered instructions with the available function names for that chat invocation Additionally, I've made some attempted improvements to simulated function calling: - simplified the system message - more emphasis on generating valid JSON (e.g. I saw multiline delimiters `"""` which are not supported) - more emphasis on not providing any input if the function does not accept any parameters. e.g. Claude was trying to provide entire search requests or SPL-like query strings as input, which led to hallucinations) There are also some other changes, which I've commented on in the file changes. **Addendum: I have pushed some more changes, related to the evaluation framework (and running it with Claude). Will comment inline in [`9ebd207` (#181058)](https://github.com/elastic/kibana/pull/181058/commits/9ebd207acd47c33077627356c464958240c9d446).** --- .../get_apm_dataset_info.ts | 1 - .../get_apm_downstream_dependencies.ts | 4 +- .../get_apm_services_list.ts | 1 - .../assistant_functions/get_apm_timeseries.ts | 1 - .../apm/server/assistant_functions/index.ts | 2 +- .../common/functions/types.ts | 9 - .../common/types.ts | 2 + .../utils/emit_with_concatenated_message.ts | 41 ++-- .../common/utils/extend_system_message.tsx | 22 -- .../utils/filter_function_definitions.ts | 8 +- ...throw_serialized_chat_completion_errors.ts | 14 +- .../message_panel/esql_code_block.tsx | 64 ++++-- .../components/message_panel/message_text.tsx | 13 +- .../public/hooks/use_chat.test.ts | 8 +- .../public/hooks/use_chat.ts | 10 +- .../public/index.ts | 2 - .../public/mock.tsx | 10 +- .../public/service/create_chat_service.ts | 17 +- .../service/create_mock_chat_service.ts | 9 +- .../service/get_assistant_system_message.ts | 28 --- .../public/storybook_mock.tsx | 9 +- .../public/types.ts | 8 +- .../public/utils/builders.ts | 2 - .../scripts/evaluation/evaluation.ts | 19 +- .../scripts/evaluation/kibana_client.ts | 188 +++++++++++------- .../evaluation/scenarios/apm/index.spec.ts | 40 ++-- .../scenarios/elasticsearch/index.spec.ts | 4 +- .../evaluation/scenarios/esql/index.spec.ts | 25 +-- .../server/functions/context.ts | 9 - .../server/functions/elasticsearch.ts | 3 +- .../server/functions/execute_connector.ts | 1 - .../functions/get_dataset_info/index.ts | 1 - .../server/functions/index.ts | 135 ++++++------- .../server/functions/kibana.ts | 1 - .../server/functions/summarize.ts | 8 +- .../server/routes/functions/route.ts | 32 ++- .../chat_function_client/index.test.ts | 6 - .../service/chat_function_client/index.ts | 31 ++- .../bedrock/process_bedrock_stream.test.ts | 4 +- .../get_system_message_instructions.ts | 23 +-- .../parse_inline_function_calls.ts | 28 +-- .../server/service/client/index.test.ts | 49 ++--- .../server/service/client/index.ts | 116 ++++++----- .../server/service/types.ts | 18 +- .../catch_function_limit_exceeded_error.ts | 66 ++++++ ...t_system_message_from_instructions.test.ts | 78 ++++++++ .../get_system_message_from_instructions.ts | 62 ++++++ .../service/util/replace_system_message.ts | 21 ++ .../components/chat/chat_body.stories.tsx | 8 +- .../components/chat/chat_flyout.stories.tsx | 4 +- .../public/utils/builders.ts | 6 +- .../public/utils/create_mock_chat_service.ts | 13 +- .../server/functions/alerts.ts | 1 - .../server/functions/changes/index.ts | 1 - .../correct_common_esql_mistakes.test.ts | 19 ++ .../query/correct_common_esql_mistakes.ts | 4 +- .../query/correct_query_with_actions.ts | 1 + .../server/functions/query/index.ts | 74 +++++-- .../functions/query/validate_esql_query.ts | 71 +++++++ .../server/functions/visualize_esql.ts | 39 +--- .../server/rule_connector/index.test.ts | 6 +- .../server/rule_connector/index.ts | 11 +- 62 files changed, 960 insertions(+), 551 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/extend_system_message.tsx delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/public/service/get_assistant_system_message.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/catch_function_limit_exceeded_error.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/replace_system_message.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts index a2821ff913f0b8..e0f3f82128dddf 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_dataset_info.ts @@ -20,7 +20,6 @@ export function registerGetApmDatasetInfoFunction({ registerFunction( { name: 'get_apm_dataset_info', - contexts: ['core'], visibility: FunctionVisibility.AssistantOnly, description: `Use this function to get information about APM data.`, parameters: { diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts index 419cdcd1e6dced..c1d1c511dac4da 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_downstream_dependencies.ts @@ -16,7 +16,6 @@ export function registerGetApmDownstreamDependenciesFunction({ registerFunction( { name: 'get_apm_downstream_dependencies', - contexts: ['core'], description: `Get the downstream dependencies (services or uninstrumented backends) for a service. This allows you to map the downstream dependency name to a service, by returning both span.destination.service.resource and service.name. Use this to @@ -39,7 +38,8 @@ export function registerGetApmDownstreamDependenciesFunction({ }, 'service.environment': { type: 'string', - description: 'The environment that the service is running in', + description: + 'The environment that the service is running in. Leave empty to query for all environments.', }, start: { type: 'string', diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts index 325d73b55070d2..1ca34117f5f9da 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_services_list.ts @@ -22,7 +22,6 @@ export function registerGetApmServicesListFunction({ registerFunction( { name: 'get_apm_services_list', - contexts: ['apm'], description: `Gets a list of services`, descriptionForUser: i18n.translate( 'xpack.apm.observabilityAiAssistant.functions.registerGetApmServicesList.descriptionForUser', diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts index 4d5cd841ff4d1b..63bdbd422c6586 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/get_apm_timeseries.ts @@ -122,7 +122,6 @@ export function registerGetApmTimeseriesFunction({ }: FunctionRegistrationParameters) { registerFunction( { - contexts: ['core'], name: 'get_apm_timeseries', description: `Visualise and analyse different APM metrics, like throughput, failure rate, or latency, for any service or all services, or any or all of its dependencies, both as a timeseries and as a single statistic. A visualisation will be displayed above your reply - DO NOT attempt to display or generate an image yourself, or any other placeholder. Additionally, the function will return any changes, such as spikes, step and trend changes, or dips. You can also use it to compare data by requesting two different time ranges, or for instance two different service versions.`, parameters, diff --git a/x-pack/plugins/observability_solution/apm/server/assistant_functions/index.ts b/x-pack/plugins/observability_solution/apm/server/assistant_functions/index.ts index eebbcf021426cd..ccc643cf4dc76c 100644 --- a/x-pack/plugins/observability_solution/apm/server/assistant_functions/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/assistant_functions/index.ts @@ -48,7 +48,7 @@ export function registerAssistantFunctions({ ruleDataClient: IRuleDataClient; plugins: APMRouteHandlerResources['plugins']; }): RegistrationCallback { - return async ({ resources, functions: { registerContext, registerFunction } }) => { + return async ({ resources, functions: { registerFunction } }) => { const apmRouteHandlerResources: MinimalAPMRouteHandlerResources = { context: resources.context, request: resources.request, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/functions/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/functions/types.ts index 12feb88bb65822..bd786e9ba3c75d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/functions/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/functions/types.ts @@ -28,11 +28,6 @@ export type CompatibleJSONSchema = { description?: string; }; -export interface ContextDefinition { - name: string; - description: string; -} - export type FunctionResponse = | { content?: any; @@ -46,10 +41,6 @@ export interface FunctionDefinition void; - -export type ContextRegistry = Map; export type FunctionRegistry = Map; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index a51db2ebc0fc6d..77e9fd33532cac 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -94,6 +94,8 @@ export interface UserInstruction { text: string; } +export type UserInstructionOrPlainText = string | UserInstruction; + export interface ObservabilityAIAssistantScreenContextRequest { screenDescription?: string; data?: Array<{ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/emit_with_concatenated_message.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/emit_with_concatenated_message.ts index b3df0af4b2eb4b..af283b78698f18 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/emit_with_concatenated_message.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/emit_with_concatenated_message.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { concat, last, mergeMap, Observable, shareReplay, withLatestFrom } from 'rxjs'; +import { concat, from, last, mergeMap, Observable, shareReplay, withLatestFrom } from 'rxjs'; import { ChatCompletionChunkEvent, MessageAddEvent, @@ -16,8 +16,32 @@ import { ConcatenatedMessage, } from './concatenate_chat_completion_chunks'; +type ConcatenateMessageCallback = ( + concatenatedMessage: ConcatenatedMessage +) => Promise; + +function mergeWithEditedMessage( + originalMessage: ConcatenatedMessage, + chunkEvent: ChatCompletionChunkEvent, + callback?: ConcatenateMessageCallback +): Observable { + return from( + (callback ? callback(originalMessage) : Promise.resolve(originalMessage)).then((message) => { + const next: MessageAddEvent = { + type: StreamingChatResponseEventType.MessageAdd as const, + id: chunkEvent.id, + message: { + '@timestamp': new Date().toISOString(), + ...message, + }, + }; + return next; + }) + ); +} + export function emitWithConcatenatedMessage( - callback?: (concatenatedMessage: ConcatenatedMessage) => Promise + callback?: ConcatenateMessageCallback ): ( source$: Observable ) => Observable { @@ -30,17 +54,8 @@ export function emitWithConcatenatedMessage( concatenateChatCompletionChunks(), last(), withLatestFrom(source$), - mergeMap(async ([message, chunkEvent]) => { - const next: MessageAddEvent = { - type: StreamingChatResponseEventType.MessageAdd as const, - id: chunkEvent.id, - message: { - '@timestamp': new Date().toISOString(), - ...(callback ? await callback(message) : message), - }, - }; - - return next; + mergeMap(([message, chunkEvent]) => { + return mergeWithEditedMessage(message, chunkEvent, callback); }) ) ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/extend_system_message.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/extend_system_message.tsx deleted file mode 100644 index 77a7d99c763eea..00000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/extend_system_message.tsx +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Message } from '../types'; - -export function extendSystemMessage(messages: Message[], extensions: string[]) { - const [systemMessage, ...rest] = messages; - - const extendedSystemMessage: Message = { - ...systemMessage, - message: { - ...systemMessage.message, - content: `${systemMessage.message.content}\n\n${extensions.join('\n\n').trim()}`, - }, - }; - - return [extendedSystemMessage].concat(rest); -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/filter_function_definitions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/filter_function_definitions.ts index 63b7661ee105ac..b9d41fb4980568 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/filter_function_definitions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/filter_function_definitions.ts @@ -8,22 +8,18 @@ import type { FunctionDefinition } from '../functions/types'; export function filterFunctionDefinitions({ - contexts, filter, definitions, }: { - contexts?: string[]; filter?: string; definitions: FunctionDefinition[]; }) { - return contexts || filter + return filter ? definitions.filter((fn) => { - const matchesContext = - !contexts || fn.contexts.some((context) => contexts.includes(context)); const matchesFilter = !filter || fn.name.includes(filter) || fn.description.includes(filter); - return matchesContext && matchesFilter; + return matchesFilter; }) : definitions; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/throw_serialized_chat_completion_errors.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/throw_serialized_chat_completion_errors.ts index 8e4718158280be..2c23109a1bac01 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/throw_serialized_chat_completion_errors.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/throw_serialized_chat_completion_errors.ts @@ -5,20 +5,21 @@ * 2.0. */ -import { filter, Observable, tap } from 'rxjs'; +import { filter, OperatorFunction, tap } from 'rxjs'; import { ChatCompletionError, ChatCompletionErrorCode, type StreamingChatResponseEvent, StreamingChatResponseEventType, type ChatCompletionErrorEvent, + BufferFlushEvent, } from '../conversation_complete'; -export function throwSerializedChatCompletionErrors() { - return ( - source$: Observable - ): Observable> => { - return source$.pipe( +export function throwSerializedChatCompletionErrors< + T extends StreamingChatResponseEvent | BufferFlushEvent +>(): OperatorFunction> { + return (source$) => + source$.pipe( tap((event) => { // de-serialise error if (event.type === StreamingChatResponseEventType.ChatCompletionError) { @@ -33,5 +34,4 @@ export function throwSerializedChatCompletionErrors() { event.type !== StreamingChatResponseEventType.ChatCompletionError ) ); - }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx index 4fe05d323572f7..24347c3aadb5ec 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx @@ -10,6 +10,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiPanel, + UseEuiTheme, useEuiTheme, } from '@elastic/eui'; import { css } from '@emotion/css'; @@ -17,6 +18,47 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { ChatActionClickHandler, ChatActionClickType } from '../chat/types'; +const getCodeBlockClassName = (theme: UseEuiTheme) => css` + background-color: ${theme.euiTheme.colors.lightestShade}; + .euiCodeBlock__pre { + margin-bottom: 0; + padding: ${theme.euiTheme.size.m}; + min-block-size: 48px; + } + .euiCodeBlock__controls { + inset-block-start: ${theme.euiTheme.size.m}; + inset-inline-end: ${theme.euiTheme.size.m}; + } +`; + +function CodeBlockWrapper({ children }: { children: React.ReactNode }) { + const theme = useEuiTheme(); + return ( + + {children} + + ); +} + +export function CodeBlock({ children }: { children: React.ReactNode }) { + return ( + + + + + {children} + + + + + ); +} + export function EsqlCodeBlock({ value, actionsDisabled, @@ -26,26 +68,8 @@ export function EsqlCodeBlock({ actionsDisabled: boolean; onActionClick: ChatActionClickHandler; }) { - const theme = useEuiTheme(); - return ( - + @@ -87,6 +111,6 @@ export function EsqlCodeBlock({ - + ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/message_text.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/message_text.tsx index 337ae9503fe658..85fa0f46099034 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/message_text.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/message_panel/message_text.tsx @@ -21,7 +21,7 @@ import type { Code, InlineCode, Parent, Text } from 'mdast'; import React, { useMemo, useRef } from 'react'; import type { Node } from 'unist'; import { ChatActionClickHandler } from '../chat/types'; -import { EsqlCodeBlock } from './esql_code_block'; +import { CodeBlock, EsqlCodeBlock } from './esql_code_block'; interface Props { content: string; @@ -104,6 +104,9 @@ const esqlLanguagePlugin = () => { if (node.type === 'code' && node.lang === 'esql') { node.type = 'esql'; + } else if (node.type === 'code') { + // switch to type that allows us to control rendering + node.type = 'codeBlock'; } }; @@ -131,6 +134,14 @@ export function MessageText({ loading, content, onActionClick }: Props) { processingPlugins[1][1].components = { ...components, cursor: Cursor, + codeBlock: (props) => { + return ( + <> + {props.value} + + + ); + }, esql: (props) => { return ( <> diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts index 838feb18330e7b..1f36b49175eea1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.test.ts @@ -26,11 +26,17 @@ const mockChatService: MockedChatService = { chat: jest.fn(), complete: jest.fn(), sendAnalyticsEvent: jest.fn(), - getContexts: jest.fn().mockReturnValue([{ name: 'core', description: '' }]), getFunctions: jest.fn().mockReturnValue([]), hasFunction: jest.fn().mockReturnValue(false), hasRenderFunction: jest.fn().mockReturnValue(true), renderFunction: jest.fn(), + getSystemMessage: jest.fn().mockReturnValue({ + '@timestamp': new Date().toISOString(), + message: { + content: 'system', + role: MessageRole.System, + }, + }), }; const addErrorMock = jest.fn(); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts index 2ab4fd294dffa8..7291557642669f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/hooks/use_chat.ts @@ -18,11 +18,7 @@ import { isTokenLimitReachedError, StreamingChatResponseEventType, } from '../../common'; -import { - getAssistantSystemMessage, - type ObservabilityAIAssistantChatService, - type ObservabilityAIAssistantService, -} from '..'; +import type { ObservabilityAIAssistantChatService, ObservabilityAIAssistantService } from '..'; import { useKibana } from './use_kibana'; import { useOnce } from './use_once'; import { useUserPreferredLanguage } from './use_user_preferred_language'; @@ -75,13 +71,11 @@ function useChatWithoutContext({ persist, }: UseChatPropsWithoutContext): UseChatResult { const [chatState, setChatState] = useState(ChatState.Ready); - const systemMessage = useMemo(() => { - return getAssistantSystemMessage({ contexts: chatService.getContexts() }); + return chatService.getSystemMessage(); }, [chatService]); useOnce(initialMessages); - useOnce(initialConversationId); const [conversationId, setConversationId] = useState(initialConversationId); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/index.ts index d42c96715523e8..52d2511f9877f9 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/index.ts @@ -57,8 +57,6 @@ export { VISUALIZE_ESQL_USER_INTENTIONS, } from '../common/functions/visualize_esql'; -export { getAssistantSystemMessage } from './service/get_assistant_system_message'; - export { isSupportedConnectorType } from '../common'; export { FunctionVisibility } from '../common'; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx index c38307e920641b..28b05433b2e1e2 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/mock.tsx @@ -9,7 +9,7 @@ import { noop } from 'lodash'; import React from 'react'; import { Observable, of } from 'rxjs'; import type { StreamingChatResponseEventWithoutError } from '../common/conversation_complete'; -import { ScreenContextActionDefinition } from '../common/types'; +import { MessageRole, ScreenContextActionDefinition } from '../common/types'; import type { ObservabilityAIAssistantAPIClient } from './api'; import type { ObservabilityAIAssistantChatService, @@ -23,7 +23,6 @@ export const mockChatService: ObservabilityAIAssistantChatService = { sendAnalyticsEvent: noop, chat: (options) => new Observable(), complete: (options) => new Observable(), - getContexts: () => [], getFunctions: () => [buildFunctionElasticsearch(), buildFunctionServiceSummary()], renderFunction: (name) => (
@@ -35,6 +34,13 @@ export const mockChatService: ObservabilityAIAssistantChatService = { ), hasFunction: () => true, hasRenderFunction: () => true, + getSystemMessage: () => ({ + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: '', + }, + }), }; export const mockService: ObservabilityAIAssistantService = { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts index 09c12004591aa3..4995aa1b584ba8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_chat_service.ts @@ -22,6 +22,7 @@ import { switchMap, timestamp, } from 'rxjs'; +import { Message, MessageRole } from '../../common'; import { type BufferFlushEvent, StreamingChatResponseEventType, @@ -106,7 +107,7 @@ export async function createChatService({ const renderFunctionRegistry: Map> = new Map(); - const [{ functionDefinitions, contextDefinitions }] = await Promise.all([ + const [{ functionDefinitions, systemMessage }] = await Promise.all([ apiClient('GET /internal/observability_ai_assistant/functions', { signal: setupAbortSignal, }), @@ -133,9 +134,7 @@ export async function createChatService({ const client: Pick = { chat(name: string, { connectorId, messages, function: callFunctions = 'auto', signal }) { return new Observable((subscriber) => { - const contexts = ['core', 'apm']; - - const functions = getFunctions({ contexts }).filter((fn) => { + const functions = getFunctions().filter((fn) => { const visibility = fn.visibility ?? FunctionVisibility.All; return ( @@ -270,7 +269,6 @@ export async function createChatService({ onActionClick, }); }, - getContexts: () => contextDefinitions, getFunctions, hasFunction: (name: string) => { return functionRegistry.has(name); @@ -278,6 +276,15 @@ export async function createChatService({ hasRenderFunction: (name: string) => { return renderFunctionRegistry.has(name); }, + getSystemMessage: (): Message => { + return { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: systemMessage, + }, + }; + }, ...client, }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_mock_chat_service.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_mock_chat_service.ts index caef109f237fcb..fc26499b1f63eb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_mock_chat_service.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/create_mock_chat_service.ts @@ -6,6 +6,7 @@ */ import type { DeeplyMockedKeys } from '@kbn/utility-types-jest'; +import { MessageRole } from '../../common'; import type { ObservabilityAIAssistantChatService } from '../types'; type MockedChatService = DeeplyMockedKeys; @@ -15,11 +16,17 @@ export const createMockChatService = (): MockedChatService => { chat: jest.fn(), complete: jest.fn(), sendAnalyticsEvent: jest.fn(), - getContexts: jest.fn().mockReturnValue([{ name: 'core', description: '' }]), getFunctions: jest.fn().mockReturnValue([]), hasFunction: jest.fn().mockReturnValue(false), hasRenderFunction: jest.fn().mockReturnValue(true), renderFunction: jest.fn(), + getSystemMessage: jest.fn().mockReturnValue({ + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: 'system', + }, + }), }; return mockChatService; }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/get_assistant_system_message.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/get_assistant_system_message.ts deleted file mode 100644 index b1050b8caa4a99..00000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/service/get_assistant_system_message.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { without } from 'lodash'; -import { MessageRole } from '../../common'; -import { ContextDefinition } from '../../common/functions/types'; -import type { Message } from '../../common/types'; - -export function getAssistantSystemMessage({ - contexts, -}: { - contexts: ContextDefinition[]; -}): Message { - const coreContext = contexts.find((context) => context.name === 'core')!; - - const otherContexts = without(contexts.concat(), coreContext); - return { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.System as const, - content: [coreContext, ...otherContexts].map((context) => context.description).join('\n'), - }, - }; -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx index 01c2f658e360b9..1d9d79838bd3aa 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/storybook_mock.tsx @@ -8,6 +8,7 @@ import { i18n } from '@kbn/i18n'; import { noop } from 'lodash'; import React from 'react'; import { Observable, of } from 'rxjs'; +import { MessageRole } from '.'; import type { StreamingChatResponseEventWithoutError } from '../common/conversation_complete'; import type { ObservabilityAIAssistantAPIClient } from './api'; import type { ObservabilityAIAssistantChatService, ObservabilityAIAssistantService } from './types'; @@ -17,7 +18,6 @@ export const createStorybookChatService = (): ObservabilityAIAssistantChatServic sendAnalyticsEvent: () => {}, chat: (options) => new Observable(), complete: (options) => new Observable(), - getContexts: () => [], getFunctions: () => [buildFunctionElasticsearch(), buildFunctionServiceSummary()], renderFunction: (name) => (
@@ -29,6 +29,13 @@ export const createStorybookChatService = (): ObservabilityAIAssistantChatServic ), hasFunction: () => true, hasRenderFunction: () => true, + getSystemMessage: () => ({ + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: '', + }, + }), }); export const createStorybookService = (): ObservabilityAIAssistantService => ({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts index 071e8e748ebf4b..0567324e164dc7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/types.ts @@ -14,11 +14,7 @@ import type { MessageAddEvent, StreamingChatResponseEventWithoutError, } from '../common/conversation_complete'; -import type { - ContextDefinition, - FunctionDefinition, - FunctionResponse, -} from '../common/functions/types'; +import type { FunctionDefinition, FunctionResponse } from '../common/functions/types'; import type { Message, ObservabilityAIAssistantScreenContext, @@ -60,9 +56,9 @@ export interface ObservabilityAIAssistantChatService { signal: AbortSignal; responseLanguage: string; }) => Observable; - getContexts: () => ContextDefinition[]; getFunctions: (options?: { contexts?: string[]; filter?: string }) => FunctionDefinition[]; hasFunction: (name: string) => boolean; + getSystemMessage: () => Message; hasRenderFunction: (name: string) => boolean; renderFunction: ( name: string, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts index a7f9df09b47a50..e233bf6da5d112 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts @@ -10,7 +10,6 @@ import type { FunctionDefinition } from '../../common/functions/types'; export function buildFunction(): FunctionDefinition { return { name: 'elasticsearch', - contexts: ['core'], description: 'Call Elasticsearch APIs on behalf of the user', descriptionForUser: 'Call Elasticsearch APIs on behalf of the user', parameters: { @@ -36,7 +35,6 @@ export const buildFunctionElasticsearch = buildFunction; export function buildFunctionServiceSummary(): FunctionDefinition { return { name: 'get_service_summary', - contexts: ['core'], description: 'Gets a summary of a single service, including: the language, service version, deployments, infrastructure, alerting, etc. ', descriptionForUser: 'Get a summary for a single service.', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/evaluation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/evaluation.ts index 24797ab42269e1..650a5cedceaa8f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/evaluation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/evaluation.ts @@ -125,7 +125,7 @@ function runEvaluations() { const mocha = new Mocha({ grep: argv.grep, - timeout: '5m', + timeout: '10m', }); const chatClient = kibanaClient.createChatClient({ @@ -253,7 +253,7 @@ function runEvaluations() { mocha.addFile(filename); } - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { mocha.run((failures: any) => { if (failures) { log.write(table.table(failedScenarios, tableConfig)); @@ -262,6 +262,21 @@ function runEvaluations() { } resolve(); }); + }).finally(() => { + const score = results + .flatMap((result) => result.scores) + .reduce( + (prev, result) => { + prev.score += result.score; + prev.total += 1; + return prev; + }, + { score: 0, total: 0 } + ); + + log.write('-------------------------------------------'); + log.write(`Scored ${score.score} out of ${score.total}`); + log.write('-------------------------------------------'); }); }, { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/kibana_client.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/kibana_client.ts index a3385be2950ff8..7604b7bca4a170 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/kibana_client.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/kibana_client.ts @@ -5,13 +5,26 @@ * 2.0. */ -import axios, { AxiosInstance, AxiosResponse } from 'axios'; -import { isArray, pick, remove } from 'lodash'; -import { concatMap, filter, lastValueFrom, toArray } from 'rxjs'; -import { format, parse, UrlObject } from 'url'; import { ToolingLog } from '@kbn/tooling-log'; +import axios, { AxiosInstance, AxiosResponse, isAxiosError } from 'axios'; +import { isArray, pick, remove } from 'lodash'; import pRetry from 'p-retry'; -import { Message, MessageRole } from '../../common'; +import { + concatMap, + defer, + filter, + from, + lastValueFrom, + of, + OperatorFunction, + retry, + switchMap, + timer, + toArray, +} from 'rxjs'; +import { format, parse, UrlObject } from 'url'; +import { inspect } from 'util'; +import { ChatCompletionErrorCode, isChatCompletionError, Message, MessageRole } from '../../common'; import { isSupportedConnectorType } from '../../common/connectors'; import { BufferFlushEvent, @@ -21,16 +34,14 @@ import { MessageAddEvent, StreamingChatResponseEvent, StreamingChatResponseEventType, - TokenCountEvent, } from '../../common/conversation_complete'; +import { FunctionDefinition } from '../../common/functions/types'; import { ObservabilityAIAssistantScreenContext } from '../../common/types'; import { concatenateChatCompletionChunks } from '../../common/utils/concatenate_chat_completion_chunks'; import { throwSerializedChatCompletionErrors } from '../../common/utils/throw_serialized_chat_completion_errors'; import { APIReturnType, ObservabilityAIAssistantAPIClientRequestParamsOf } from '../../public'; -import { getAssistantSystemMessage } from '../../public/service/get_assistant_system_message'; import { streamIntoObservable } from '../../server/service/util/stream_into_observable'; import { EvaluationResult } from './types'; -import { FunctionDefinition } from '../../common/functions/types'; // eslint-disable-next-line spaced-comment /// @@ -170,13 +181,13 @@ export class KibanaClient { async function getFunctions() { const { - data: { functionDefinitions, contextDefinitions }, + data: { functionDefinitions }, }: AxiosResponse> = await that.axios.get( that.getUrl({ pathname: '/internal/observability_ai_assistant/functions' }) ); - return { functionDefinitions, contextDefinitions }; + return { functionDefinitions }; } let currentTitle: string = ''; @@ -202,6 +213,61 @@ export class KibanaClient { unregister: () => void; }> = []; + function serializeAndHandleRetryableErrors< + T extends StreamingChatResponseEvent + >(): OperatorFunction> { + return (source$) => { + const processed$ = source$.pipe( + concatMap((buffer: Buffer) => + buffer + .toString('utf-8') + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as T | BufferFlushEvent) + ), + throwSerializedChatCompletionErrors(), + retry({ + count: 1, + delay: (error) => { + that.log.error('Error in stream'); + + if (isAxiosError(error)) { + that.log.error( + inspect( + { + message: error.message, + status: error.status, + response: error.response?.data, + }, + { depth: 10 } + ) + ); + } else { + that.log.error(inspect(error, { depth: 10 })); + } + + if ( + isChatCompletionError(error) && + error.code !== ChatCompletionErrorCode.InternalError + ) { + that.log.info(`Not retrying error ${error.code}`); + return of(); + } + that.log.info(`Retrying in 5s`); + return timer(5000); + }, + }), + filter( + (event): event is Exclude => + event.type !== StreamingChatResponseEventType.BufferFlush + ) + ); + + return processed$; + }; + } + async function chat( name: string, { @@ -216,46 +282,37 @@ export class KibanaClient { connectorIdOverride?: string; } ) { - const params: ObservabilityAIAssistantAPIClientRequestParamsOf<'POST /internal/observability_ai_assistant/chat'>['params']['body'] = - { - name, - messages, - connectorId: connectorIdOverride || connectorId, - functions: functions.map((fn) => pick(fn, 'name', 'description', 'parameters')), - functionCall, - }; - const stream$ = streamIntoObservable( - ( - await that.axios.post( - that.getUrl({ - pathname: '/internal/observability_ai_assistant/chat', - }), - params, - { responseType: 'stream', timeout: NaN } - ) - ).data - ).pipe( - concatMap((buffer: Buffer) => - buffer - .toString('utf-8') - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .map( - (line) => - JSON.parse(line) as StreamingChatResponseEvent | BufferFlushEvent | TokenCountEvent - ) - ), + that.log.info('Chat', name); + + const chat$ = defer(() => { + that.log.debug(`Calling chat API`); + const params: ObservabilityAIAssistantAPIClientRequestParamsOf<'POST /internal/observability_ai_assistant/chat'>['params']['body'] = + { + name, + messages, + connectorId: connectorIdOverride || connectorId, + functions: functions.map((fn) => pick(fn, 'name', 'description', 'parameters')), + functionCall, + }; + + return that.axios.post( + that.getUrl({ + pathname: '/internal/observability_ai_assistant/chat', + }), + params, + { responseType: 'stream', timeout: NaN } + ); + }).pipe( + switchMap((response) => streamIntoObservable(response.data)), + serializeAndHandleRetryableErrors(), filter( - (line): line is ChatCompletionChunkEvent | ChatCompletionErrorEvent => - line.type === StreamingChatResponseEventType.ChatCompletionChunk || - line.type === StreamingChatResponseEventType.ChatCompletionError + (line): line is ChatCompletionChunkEvent => + line.type === StreamingChatResponseEventType.ChatCompletionChunk ), - throwSerializedChatCompletionErrors(), concatenateChatCompletionChunks() ); - const message = await lastValueFrom(stream$); + const message = await lastValueFrom(chat$); return message.message; } @@ -264,9 +321,8 @@ export class KibanaClient { return { chat: async (message) => { - const { functionDefinitions, contextDefinitions } = await getFunctions(); + const { functionDefinitions } = await getFunctions(); const messages = [ - getAssistantSystemMessage({ contexts: contextDefinitions }), ...getMessages(message).map((msg) => ({ message: msg, '@timestamp': new Date().toISOString(), @@ -275,6 +331,7 @@ export class KibanaClient { return chat('chat', { messages, functions: functionDefinitions }); }, complete: async (...args) => { + that.log.info(`Complete`); let messagesArg: StringOrMessageList; let conversationId: string | undefined; let options: Options = {}; @@ -301,18 +358,17 @@ export class KibanaClient { options = args[2]; } - const { contextDefinitions } = await getFunctions(); const messages = [ - getAssistantSystemMessage({ contexts: contextDefinitions }), ...getMessages(messagesArg!).map((msg) => ({ message: msg, '@timestamp': new Date().toISOString(), })), ]; - const stream$ = streamIntoObservable( - ( - await that.axios.post( + const stream$ = defer(() => { + that.log.debug(`Calling /chat/complete API`); + return from( + that.axios.post( that.getUrl({ pathname: '/internal/observability_ai_assistant/chat/complete', }), @@ -326,28 +382,17 @@ export class KibanaClient { }, { responseType: 'stream', timeout: NaN } ) - ).data - ).pipe( - concatMap((buffer: Buffer) => - buffer - .toString('utf-8') - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .map( - (line) => - JSON.parse(line) as - | StreamingChatResponseEvent - | BufferFlushEvent - | TokenCountEvent - ) - ), + ); + }).pipe( + switchMap((response) => { + return streamIntoObservable(response.data); + }), + serializeAndHandleRetryableErrors(), filter( (event): event is MessageAddEvent | ConversationCreateEvent => event.type === StreamingChatResponseEventType.MessageAdd || event.type === StreamingChatResponseEventType.ConversationCreate ), - throwSerializedChatCompletionErrors(), toArray() ); @@ -401,7 +446,9 @@ export class KibanaClient { This is the conversation: - ${JSON.stringify(messages)}`, + ${JSON.stringify( + messages.map((msg) => pick(msg, 'content', 'name', 'function_call', 'role')) + )}`, }, }, ], @@ -437,7 +484,6 @@ export class KibanaClient { }, required: ['criteria'], }, - contexts: [], description: 'Call this function to return scores for the criteria', }, ], diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/apm/index.spec.ts index 24a1b21f952370..6d715dd911dbfa 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/apm/index.spec.ts @@ -91,33 +91,31 @@ describe('apm', () => { ); }); - it('service summary, troughput, dependencies and errors', async () => { - let conversation = await chatClient.complete( - 'What is the status of the service ai-assistant-service in the test environment?' + it('service throughput', async () => { + const conversation = await chatClient.complete( + 'What is the average throughput per minute for the ai-assistant-service service over the past 4 hours?' ); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ - content: - 'What is the average throughput for the ai-assistant-service service over the past 4 hours?', - role: MessageRole.User, - }) - ); + const result = await chatClient.evaluate(conversation, [ + 'Uses the get_apm_dataset_info function to get information about the APM data streams', + 'Uses the query function to generate an ES|QL query', + 'Generates a valid ES|QL query that returns the throughput over the past 4 hours.', + 'Uses the execute_query function to get the results for the generated query', + 'Summarizes the results for the user', + 'Calculates a throughput of 30 transactions per minute', + ]); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ - content: 'What are the downstream dependencies of the ai-assistant-service-front service?', - role: MessageRole.User, - }) + expect(result.passed).to.be(true); + }); + + it('service dependencies', async () => { + const conversation = await chatClient.complete( + 'What are the downstream dependencies of the ai-assistant-service-front service?' ); const result = await chatClient.evaluate(conversation, [ - 'Uses get_apm_service_summary to obtain the status of the ai-assistant-service service', - 'Executes get_apm_timeseries to obtain the throughput of the services ai-assistant-service for the last 4 hours', - 'Gives a summary of the throughput stats for ai-assistant-service', - 'Provides the downstream dependencies of ai-assistant-service-front', + 'Uses the get_apm_downstream_dependencies function with the `service.name` parameter being "ai-assistant-service-front"', + 'Returns the results to the user ("ai-assistant-service-back" is the only dependency)', ]); expect(result.passed).to.be(true); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 1dba311a644959..20f78d487a4bf1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -61,8 +61,8 @@ describe('elasticsearch functions', () => { const conversation = await chatClient.complete('How many documents are in the index kb?'); const result = await chatClient.evaluate(conversation, [ - 'Calls the Elasticsearch function', - 'Finds how many documents are in that index', + 'Calls the `elasticsearch` function OR the `query` function', + 'Finds how many documents are in that index (one document)', ]); expect(result.passed).to.be(true); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/esql/index.spec.ts index 4dfaa6df835c1f..3bd6fc6a0c2075 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/scripts/evaluation/scenarios/esql/index.spec.ts @@ -34,7 +34,7 @@ async function evaluateEsqlQuery({ : []), ...(execute ? [`The query successfully executed without an error`] - : [`The query was not executed`]), + : [`The query was not executed, it was only explained`]), ...criteria, ]); @@ -154,7 +154,8 @@ describe('ES|QL query generation', () => { question: 'From employees, I want to sort the documents by salary, and then return 10 results per page, and then see the second page', criteria: [ - 'The assistant should mention that pagination is currently not supported in ES|QL', + 'The assistant should clearly mention that pagination is currently not supported in ES|QL', + 'IF the assistant decides to execute the query, it should correctly execute, and the Assistant should clearly mention pagination is not currently supported', ], }); }); @@ -180,7 +181,7 @@ describe('ES|QL query generation', () => { it('logs avg cpu', async () => { await evaluateEsqlQuery({ question: - 'My metric data (ECS) is in .ds-metrics-apm* Show me a query that gets the average CPU per service, limit it to the top 10 results, in 1m buckets, and only include the last 15m. ', + 'My metrics data is in `metrics-*`. I want to see what a query would look like that gets the average CPU per service, limit it to the top 10 results, in 1m buckets, and only include the last 15m.', expected: `FROM .ds-metrics-apm* | WHERE @timestamp >= NOW() - 15 minutes | EVAL bucket = DATE_TRUNC(1 minute, @timestamp) @@ -193,7 +194,7 @@ describe('ES|QL query generation', () => { it('metricbeat avg cpu', async () => { await evaluateEsqlQuery({ - question: `From metricbeat*, using ES|QL, show me a query to see the percentage of CPU time (system.cpu.system.pct) normalized by the number of CPU cores (system.cpu.cores), broken down by hostname`, + question: `Assume my data is in \`metricbeat*\`. Show me a query to see the percentage of CPU time (system.cpu.system.pct) normalized by the number of CPU cores (system.cpu.cores), broken down by host name`, expected: `FROM metricbeat* | EVAL system_pct_normalized = TO_DOUBLE(system.cpu.system.pct) / system.cpu.cores | STATS avg_system_pct_normalized = AVG(system_pct_normalized) BY host.name @@ -205,7 +206,7 @@ describe('ES|QL query generation', () => { it('postgres avg duration dissect', async () => { await evaluateEsqlQuery({ question: - 'Show me an ESQL query to extract the query duration from postgres log messages in postgres-logs*, with this format "2021-01-01 00:00:00 UTC [12345]: [1-1] user=postgres,db=mydb,app=[unknown],client=127.0.0.1 LOG: duration: 123.456 ms statement: SELECT * FROM my_table", using ECS fields, and calculate the avg', + 'Show me an example ESQL query to extract the query duration from postgres log messages in postgres-logs*, with this format:\n `2021-01-01 00:00:00 UTC [12345]: [1-1] user=postgres,db=mydb,app=[unknown],client=127.0.0.1 LOG: duration: 123.456 ms statement: SELECT * FROM my_table`. \n Use ECS fields, and calculate the avg.', expected: `FROM postgres-logs* | DISSECT message "%{}: duration: %{query_duration} ms %{}" | EVAL duration_double = TO_DOUBLE(duration) @@ -256,14 +257,12 @@ describe('ES|QL query generation', () => { ); }); - it('metrics avg duration', async () => { + // histograms are not supported yet in ES|QL + it.skip('metrics avg duration', async () => { await evaluateEsqlQuery({ question: 'Execute a query for metrics-apm*, filtering on metricset.name:service_transaction and metricset.interval:1m, the average duration (via transaction.duration.histogram), in 50 buckets.', execute: true, - criteria: [ - 'The assistant know that transaction.duration.histogram cannot be used in ESQL and proposes an alertative solution', - ], }); }); @@ -274,8 +273,7 @@ describe('ES|QL query generation', () => { expected: `FROM traces-apm* | WHERE @timestamp >= NOW() - 24 hours | EVAL is_failure = CASE(event.outcome == "failure", 1, 0), is_success = CASE(event.outcome == "success", 1, 0) - | STATS total_requests = COUNT(*), avg_duration = AVG(transaction.duration.us), total_failures = SUM(is_failure), total_success = SUM(is_success) BY service.name - | EVAL success_rate = total_success / (total_failures + total_success) + | STATS total_requests = COUNT(*), avg_duration = AVG(transaction.duration.us), success_rate = SUM(is_success) / COUNT(*) BY service.name | KEEP service.name, avg_duration, success_rate, total_requests`, execute: true, }); @@ -328,9 +326,12 @@ describe('ES|QL query generation', () => { expected: `FROM logs-apm* | SORT @timestamp DESC | EVAL formatted_date = DATE_FORMAT("hh:mm a, d 'of' MMMM yyyy", @timestamp) - | KEEP formatted_date, log.level, message + | KEEP formatted_date, processor.event, message | LIMIT 5`, execute: true, + criteria: [ + 'The Assistant uses KEEP, to make sure the AT LEAST the formatted date, processor event and message fields are displayed. More columns are fine, fewer are not', + ], }); }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts index 15d34d36a4ffc2..7c785392dfaf48 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts @@ -35,7 +35,6 @@ export function registerContextFunction({ functions.registerFunction( { name: 'context', - contexts: ['core'], description: 'This function provides context as to what the user is looking at on their screen, and recalled documents from the knowledge base that matches their query', visibility: FunctionVisibility.AssistantOnly, @@ -68,10 +67,6 @@ export function registerContextFunction({ const { queries, categories } = args; async function getContext() { - const systemMessage = messages.find( - (message) => message.message.role === MessageRole.System - ); - const screenDescription = compact( screenContexts.map((context) => context.screenDescription) ).join('\n\n'); @@ -93,10 +88,6 @@ export function registerContextFunction({ return { content }; } - if (!systemMessage) { - throw new Error('No system message found'); - } - const userMessage = last( messages.filter((message) => message.message.role === MessageRole.User) ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts index 05900a2d475558..61a8b6adf3ed37 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts @@ -14,7 +14,6 @@ export function registerElasticsearchFunction({ functions.registerFunction( { name: 'elasticsearch', - contexts: ['core'], description: 'Call Elasticsearch APIs on behalf of the user. Make sure the request body is valid for the API that you are using. Only call this function when the user has explicitly requested it.', descriptionForUser: 'Call Elasticsearch APIs on behalf of the user', @@ -47,7 +46,7 @@ export function registerElasticsearchFunction({ body, }); - return { content: response }; + return { content: { response } }; } ); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts index 8e668d0295ba5f..0088e35a6f6af0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/execute_connector.ts @@ -15,7 +15,6 @@ export function registerExecuteConnectorFunction({ functions.registerFunction( { name: 'execute_connector', - contexts: ['core'], description: 'Use this function when user explicitly asks to call a kibana connector.', visibility: FunctionVisibility.AssistantOnly, parameters: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts index bfd0ef43d546c1..1554df10175a22 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/get_dataset_info/index.ts @@ -16,7 +16,6 @@ export function registerGetDatasetInfoFunction({ functions.registerFunction( { name: 'get_dataset_info', - contexts: ['core'], visibility: FunctionVisibility.AssistantOnly, description: `Use this function to get information about indices/datasets available and the fields available on them. diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts index c24c8d9f38803e..7f706046a693ca 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/index.ts @@ -34,97 +34,84 @@ export const registerFunctions: RegistrationCallback = async ({ const isServerless = !!resources.plugins.serverless; - return client.getKnowledgeBaseStatus().then((response) => { - const isReady = response.ready; + functions.registerInstruction(`You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems. You can help them visualise and analyze data, investigate their systems, perform root cause analysis or identify optimisation opportunities. - let description = dedent( - `You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems. You can help them visualise and analyze data, investigate their systems, perform root cause analysis or identify optimisation opportunities. + It's very important to not assume what the user is meaning. Ask them for clarification if needed. - It's very important to not assume what the user is meaning. Ask them for clarification if needed. + If you are unsure about which function should be used and with what arguments, ask the user for clarification or confirmation. - If you are unsure about which function should be used and with what arguments, ask the user for clarification or confirmation. + In KQL ("kqlFilter")) escaping happens with double quotes, not single quotes. Some characters that need escaping are: ':()\\\ + /\". Always put a field value in double quotes. Best: service.name:\"opbeans-go\". Wrong: service.name:opbeans-go. This is very important! - In KQL ("kqlFilter")) escaping happens with double quotes, not single quotes. Some characters that need escaping are: ':()\\\ - /\". Always put a field value in double quotes. Best: service.name:\"opbeans-go\". Wrong: service.name:opbeans-go. This is very important! + You can use Github-flavored Markdown in your responses. If a function returns an array, consider using a Markdown table to format the response. + + Note that ES|QL (the Elasticsearch Query Language which is a new piped language) is the preferred query language. - You can use Github-flavored Markdown in your responses. If a function returns an array, consider using a Markdown table to format the response. + DO NOT UNDER ANY CIRCUMSTANCES USE ES|QL syntax (\`service.name == "foo"\`) with "kqlFilter" (\`service.name:"foo"\`). + + The user is able to change the language which they want you to reply in on the settings page of the AI Assistant for Observability, which can be found in the ${ + isServerless ? `Project settings.` : `Stack Management app under the option AI Assistants` + }. + If the user asks how to change the language, reply in the same language the user asked in.`); - Note that ES|QL (the Elasticsearch Query Language which is a new piped language) is the preferred query language. + const { ready: isReady } = await client.getKnowledgeBaseStatus(); - You MUST use the "query" function when the user wants to: - - visualize data - - run any arbitrary query - - breakdown or filter ES|QL queries that are displayed on the current page - - convert queries from another language to ES|QL - - asks general questions about ES|QL + functions.registerInstruction(({ availableFunctionNames }) => { + const instructions: string[] = []; - DO NOT UNDER ANY CIRCUMSTANCES generate ES|QL queries or explain anything about the ES|QL query language yourself. - DO NOT UNDER ANY CIRCUMSTANCES try to correct an ES|QL query yourself - always use the "query" function for this. - - DO NOT UNDER ANY CIRCUMSTANCES USE ES|QL syntax (\`service.name == "foo"\`) with "kqlFilter" (\`service.name:"foo"\`). - - Even if the "context" function was used before that, follow it up with the "query" function. If a query fails, do not attempt to correct it yourself. Again you should call the "query" function, - even if it has been called before. - - When the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt. - If the "execute_query" function has been called, summarize these results for the user. The user does not see a visualization in this case. - - You MUST use the get_dataset_info function ${ - functions.hasFunction('get_apm_dataset_info') ? 'or get_apm_dataset_info' : '' - } function before calling the "query" or "changes" function. - - If a function requires an index, you MUST use the results from the dataset info functions. + if (availableFunctionNames.includes('get_dataset_info')) { + instructions.push(`You MUST use the get_dataset_info function ${ + functions.hasFunction('get_apm_dataset_info') ? 'or get_apm_dataset_info' : '' + } function before calling the "query" or "changes" function. + + If a function requires an index, you MUST use the results from the dataset info functions.`); + } - ${ - functions.hasFunction('get_data_on_screen') - ? `You have access to data on the screen by calling the "get_data_on_screen" function. + if (availableFunctionNames.includes('get_data_on_screen')) { + instructions.push(`You have access to data on the screen by calling the "get_data_on_screen" function. Use it to help the user understand what they are looking at. A short summary of what they are looking at is available in the return of the "context" function. - Data that is compact enough automatically gets included in the response for the "context" function. - ` - : '' - } - - The user is able to change the language which they want you to reply in on the settings page of the AI Assistant for Observability, which can be found in the ${ - isServerless ? `Project settings.` : `Stack Management app under the option AI Assistants` - }. - If the user asks how to change the language, reply in the same language the user asked in. - ` - ); + Data that is compact enough automatically gets included in the response for the "context" function.`); + } if (isReady) { - description += `You can use the "summarize" functions to store new information you have learned in a knowledge database. Once you have established that you did not know the answer to a question, and the user gave you this information, it's important that you create a summarisation of what you have learned and store it in the knowledge database. Don't create a new summarization if you see a similar summarization in the conversation, instead, update the existing one by re-using its ID. - All summaries MUST be created in English, even if the conversation was carried out in a different language. - - Additionally, you can use the "context" function to retrieve relevant information from the knowledge database. - - `; - - registerSummarizationFunction(registrationParameters); + if (availableFunctionNames.includes('summarize')) { + instructions.push(`You can use the "summarize" functions to store new information you have learned in a knowledge database. + Only use this function when the user asks for it. + All summaries MUST be created in English, even if the conversation was carried out in a different language.`); + } + + if (availableFunctionNames.includes('context')) { + instructions.push( + `Additionally, you can use the "context" function to retrieve relevant information from the knowledge database.` + ); + } } else { - description += `You do not have a working memory. If the user expects you to remember the previous conversations, tell them they can set up the knowledge base.`; + instructions.push( + `You do not have a working memory. If the user expects you to remember the previous conversations, tell them they can set up the knowledge base.` + ); } + return instructions.map((instruction) => dedent(instruction)); + }); - registerContextFunction({ ...registrationParameters, isKnowledgeBaseAvailable: isReady }); + if (isReady) { + registerSummarizationFunction(registrationParameters); + } - registerElasticsearchFunction(registrationParameters); - const request = registrationParameters.resources.request; + registerContextFunction({ ...registrationParameters, isKnowledgeBaseAvailable: isReady }); - if ('id' in request) { - registerKibanaFunction({ - ...registrationParameters, - resources: { - ...registrationParameters.resources, - request, - }, - }); - } - registerGetDatasetInfoFunction(registrationParameters); + registerElasticsearchFunction(registrationParameters); + const request = registrationParameters.resources.request; - registerExecuteConnectorFunction(registrationParameters); - - functions.registerContext({ - name: 'core', - description: dedent(description), + if ('id' in request) { + registerKibanaFunction({ + ...registrationParameters, + resources: { + ...registrationParameters.resources, + request, + }, }); - }); + } + registerGetDatasetInfoFunction(registrationParameters); + + registerExecuteConnectorFunction(registrationParameters); }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts index 4af123a43f8917..f939e3a79799b3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/kibana.ts @@ -20,7 +20,6 @@ export function registerKibanaFunction({ functions.registerFunction( { name: 'kibana', - contexts: ['core'], description: 'Call Kibana APIs on behalf of the user. Only call this function when the user has explicitly requested it, and you know how to call it, for example by querying the knowledge base or having the user explain it to you. Assume that pathnames, bodies and query parameters may have changed since your knowledge cut off date.', descriptionForUser: 'Call Kibana APIs on behalf of the user', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 39b73aefa89f69..4ff8e3ee4da91d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -15,9 +15,11 @@ export function registerSummarizationFunction({ functions.registerFunction( { name: 'summarize', - contexts: ['core'], - description: - "Use this function to summarize things learned from the conversation. You can score the learnings with a confidence metric, whether it is a correction on a previous learning. An embedding will be created that you can recall later with a semantic search. There is no need to ask the user for permission to store something you have learned, unless you do not feel confident. When you create this summarisation, make sure you craft it in a way that can be recalled with a semantic search later, and that it would have answered the user's original request.", + description: `Use this function to store facts in the knowledge database if the user requests it. + You can score the learnings with a confidence metric, whether it is a correction on a previous learning. + An embedding will be created that you can recall later with a semantic search. + When you create this summarisation, make sure you craft it in a way that can be recalled with a semantic + search later, and that it would have answered the user's original request.`, descriptionForUser: 'This function allows the Elastic Assistant to summarize things from the conversation.', parameters: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index 9b48dc3f472d10..58c93737b66172 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -7,9 +7,10 @@ import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; -import { ContextDefinition, FunctionDefinition } from '../../../common/functions/types'; +import { FunctionDefinition } from '../../../common/functions/types'; import { KnowledgeBaseEntryRole } from '../../../common/types'; import type { RecalledEntry } from '../../service/knowledge_base_service'; +import { getSystemMessageFromInstructions } from '../../service/util/get_system_message_from_instructions'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; const getFunctionsRoute = createObservabilityAIAssistantServerRoute({ @@ -21,7 +22,7 @@ const getFunctionsRoute = createObservabilityAIAssistantServerRoute({ resources ): Promise<{ functionDefinitions: FunctionDefinition[]; - contextDefinitions: ContextDefinition[]; + systemMessage: string; }> => { const { service, request } = resources; @@ -32,16 +33,29 @@ const getFunctionsRoute = createObservabilityAIAssistantServerRoute({ const client = await service.getClient({ request }); - const functionClient = await service.getFunctionClient({ - signal: controller.signal, - resources, - client, - screenContexts: [], - }); + const [functionClient, knowledgeBaseInstructions] = await Promise.all([ + service.getFunctionClient({ + signal: controller.signal, + resources, + client, + screenContexts: [], + }), + // error is caught in client + client.fetchKnowledgeBaseInstructions(), + ]); + + const functionDefinitions = functionClient.getFunctions().map((fn) => fn.definition); + + const availableFunctionNames = functionDefinitions.map((def) => def.name); return { functionDefinitions: functionClient.getFunctions().map((fn) => fn.definition), - contextDefinitions: functionClient.getContexts(), + systemMessage: getSystemMessageFromInstructions({ + registeredInstructions: functionClient.getInstructions(), + knowledgeBaseInstructions, + requestInstructions: [], + availableFunctionNames, + }), }; }, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts index a55098c5ec4fe3..9ecbd450cba301 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.test.ts @@ -20,14 +20,9 @@ describe('chatFunctionClient', () => { }); client = new ChatFunctionClient([]); - client.registerContext({ - description: '', - name: 'core', - }); client.registerFunction( { - contexts: ['core'], description: '', name: 'myFunction', parameters: { @@ -93,7 +88,6 @@ describe('chatFunctionClient', () => { expect(functions[0]).toEqual({ definition: { - contexts: ['core'], description: expect.any(String), name: 'get_data_on_screen', parameters: expect.any(Object), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts index ea09024a137e76..d0b019d635c12c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/chat_function_client/index.ts @@ -9,16 +9,17 @@ import Ajv, { type ErrorObject, type ValidateFunction } from 'ajv'; import dedent from 'dedent'; import { compact, keyBy } from 'lodash'; -import { - FunctionVisibility, - type ContextDefinition, - type ContextRegistry, - type FunctionResponse, - type RegisterContextDefinition, -} from '../../../common/functions/types'; +import { FunctionVisibility, type FunctionResponse } from '../../../common/functions/types'; import type { Message, ObservabilityAIAssistantScreenContextRequest } from '../../../common/types'; import { filterFunctionDefinitions } from '../../../common/utils/filter_function_definitions'; -import type { ChatFn, FunctionHandler, FunctionHandlerRegistry, RegisterFunction } from '../types'; +import type { + ChatFn, + FunctionHandler, + FunctionHandlerRegistry, + RegisteredInstruction, + RegisterFunction, + RegisterInstruction, +} from '../types'; export class FunctionArgsValidationError extends Error { constructor(public readonly errors: ErrorObject[]) { @@ -31,7 +32,7 @@ const ajv = new Ajv({ }); export class ChatFunctionClient { - private readonly contextRegistry: ContextRegistry = new Map(); + private readonly instructions: RegisteredInstruction[] = []; private readonly functionRegistry: FunctionHandlerRegistry = new Map(); private readonly validators: Map = new Map(); @@ -46,7 +47,6 @@ export class ChatFunctionClient { this.registerFunction( { name: 'get_data_on_screen', - contexts: ['core'], description: dedent(`Get data that is on the screen: ${allData.map((data) => `${data.name}: ${data.description}`).join('\n')} `), @@ -89,8 +89,8 @@ export class ChatFunctionClient { this.functionRegistry.set(definition.name, { definition, respond }); }; - registerContext: RegisterContextDefinition = (context) => { - this.contextRegistry.set(context.name, context); + registerInstruction: RegisterInstruction = (instruction) => { + this.instructions.push(instruction); }; validate(name: string, parameters: unknown) { @@ -105,8 +105,8 @@ export class ChatFunctionClient { } } - getContexts(): ContextDefinition[] { - return Array.from(this.contextRegistry.values()); + getInstructions(): RegisteredInstruction[] { + return this.instructions; } hasAction(name: string) { @@ -114,10 +114,8 @@ export class ChatFunctionClient { } getFunctions({ - contexts, filter, }: { - contexts?: string[]; filter?: string; } = {}): FunctionHandler[] { const allFunctions = Array.from(this.functionRegistry.values()); @@ -125,7 +123,6 @@ export class ChatFunctionClient { const functionsByName = keyBy(allFunctions, (definition) => definition.definition.name); const matchingDefinitions = filterFunctionDefinitions({ - contexts, filter, definitions: allFunctions.map((fn) => fn.definition), }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/bedrock/process_bedrock_stream.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/bedrock/process_bedrock_stream.test.ts index ee426db5153225..90f7d6f5ee69c7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/bedrock/process_bedrock_stream.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/bedrock/process_bedrock_stream.test.ts @@ -173,7 +173,9 @@ describe('processBedrockStream', () => { ); } - await expect(fn).rejects.toThrowErrorMatchingInlineSnapshot(`"no elements in sequence"`); + await expect(fn).rejects.toThrowErrorMatchingInlineSnapshot( + `"Unexpected token 'i', \\"invalid json\\" is not valid JSON"` + ); }); it('successfully invokes a function without parameters', async () => { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts index dd74052ce7bbe4..dae509e169e10d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/get_system_message_instructions.ts @@ -16,26 +16,18 @@ export function getSystemMessageInstructions({ if (functions?.length) { return `In this environment, you have access to a set of tools you can use to answer the user's question. - When deciding what tool to use, keep in mind that you can call other tools in successive requests, so decide what tool - would be a good first step. - - You MUST only invoke a single tool, and invoke it once. Other invocations will be ignored. - You MUST wait for the results before invoking another. - You can call multiple tools in successive messages. This means you can chain tool calls. If any tool was used in a previous - message, consider whether it still makes sense to follow it up with another tool call. - ${ functions?.find((fn) => fn.name === 'context') ? `The "context" tool is ALWAYS used after a user question. Even if it was used before, your job is to answer the last user question, even if the "context" tool was executed after that. Consider the tools you need to answer the user's question.` : '' } - - Rather than explaining how you would call a tool, just generate the JSON to call the tool. It will automatically be - executed and returned to you. - These results are generally not visible to the user. Treat them as if they are not, - unless specified otherwise. + DO NOT call a tool when it is not listed. + ONLY define input that is defined in the tool properties. + If a tool does not have properties, leave them out. + + It is EXTREMELY important that you generate valid JSON between the \`\`\`json and \`\`\` delimiters. You may call them like this. @@ -77,7 +69,8 @@ export function getSystemMessageInstructions({ ${TOOL_USE_START} \`\`\`json { - "name": "my_tool_without_parameters" + "name": "my_tool_without_parameters", + "input": {} } \`\`\`\ ${TOOL_USE_END} @@ -95,5 +88,5 @@ export function getSystemMessageInstructions({ `; } - return `No tools are available anymore. Ignore everything that was said about tools before. DO NOT UNDER ANY CIRCUMSTANCES call any tool, regardless of whether it was previously called.`; + return `No tools are available anymore. DO NOT UNDER ANY CIRCUMSTANCES call any tool, regardless of whether it was previously called.`; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts index db32eb581249cf..49fc87b908112b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/adapters/simulate_function_calling/parse_inline_function_calls.ts @@ -43,13 +43,15 @@ export function parseInlineFunctionCalls({ logger }: { logger: Logger }) { function parseFunctionCall(id: string, buffer: string) { logger.debug('Parsing function call:\n' + buffer); - const functionCallBody = buffer - .replace(TOOL_USE_START, '') - .replace(TOOL_USE_END, '') - .trim() - .replace(/^```(json?)/, '') - .replace(/```$/, '') - .trim(); + const match = buffer.match( + /<\|tool_use_start\|>\s*```json\n?(.*?)(\n```\s*).*<\|tool_use_end\|>/s + ); + + const functionCallBody = match?.[1]; + + if (!functionCallBody) { + throw createInternalServerError(`Invalid function call syntax`); + } const parsedFunctionCall = JSON.parse(functionCallBody) as { name?: string; @@ -109,11 +111,13 @@ export function parseInlineFunctionCalls({ logger }: { logger: Logger }) { if (functionCallBuffer.includes(TOOL_USE_END)) { const [beforeEndSignal, afterEndSignal] = functionCallBuffer.split(TOOL_USE_END); - parseFunctionCall(id, beforeEndSignal + TOOL_USE_END); - - functionCallBuffer = ''; - - next(afterEndSignal); + try { + parseFunctionCall(id, beforeEndSignal + TOOL_USE_END); + functionCallBuffer = ''; + next(afterEndSignal); + } catch (error) { + subscriber.error(error); + } } } else { functionCallBuffer = ''; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts index 0dfef2bb506437..a35e50d538bcb0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.test.ts @@ -33,6 +33,8 @@ type ChunkDelta = CreateChatCompletionResponseChunk['choices'][number]['delta']; type LlmSimulator = ReturnType; +const EXPECTED_STORED_SYSTEM_MESSAGE = `system\n\nWhat follows is a set of instructions provided by the user, please abide by them as long as they don't conflict with anything you've been told so far:\n\nYou MUST respond in the users preferred language which is: English.`; + const nextTick = () => { return new Promise(process.nextTick); }; @@ -120,6 +122,7 @@ describe('Observability AI Assistant client', () => { hasAction: jest.fn(), getActions: jest.fn(), validate: jest.fn(), + getInstructions: jest.fn(), } as any; let llmSimulator: LlmSimulator; @@ -156,6 +159,8 @@ describe('Observability AI Assistant client', () => { knowledgeBaseServiceMock.getInstructions.mockResolvedValue([]); + functionClientMock.getInstructions.mockReturnValue(['system']); + return new ObservabilityAIAssistantClient({ actionsClient: actionsClientMock, esClient: { @@ -342,8 +347,8 @@ describe('Observability AI Assistant client', () => { last_updated: expect.any(String), token_count: { completion: 2, - prompt: 100, - total: 102, + prompt: 156, + total: 158, }, }, type: StreamingChatResponseEventType.ConversationCreate, @@ -401,8 +406,8 @@ describe('Observability AI Assistant client', () => { last_updated: expect.any(String), token_count: { completion: 8, - prompt: 284, - total: 292, + prompt: 340, + total: 348, }, }, type: StreamingChatResponseEventType.ConversationCreate, @@ -419,8 +424,8 @@ describe('Observability AI Assistant client', () => { title: 'An auto-generated title', token_count: { completion: 8, - prompt: 284, - total: 292, + prompt: 340, + total: 348, }, }, labels: {}, @@ -434,8 +439,7 @@ describe('Observability AI Assistant client', () => { { '@timestamp': expect.any(String), message: { - content: - 'This is a system message\n\nYou MUST respond in the users preferred language which is: English.', + content: EXPECTED_STORED_SYSTEM_MESSAGE, role: MessageRole.System, }, }, @@ -546,8 +550,8 @@ describe('Observability AI Assistant client', () => { last_updated: expect.any(String), token_count: { completion: 2, - prompt: 100, - total: 102, + prompt: 156, + total: 158, }, }, type: StreamingChatResponseEventType.ConversationUpdate, @@ -565,8 +569,8 @@ describe('Observability AI Assistant client', () => { title: 'My stored conversation', token_count: { completion: 2, - prompt: 100, - total: 102, + prompt: 156, + total: 158, }, }, labels: {}, @@ -580,8 +584,7 @@ describe('Observability AI Assistant client', () => { { '@timestamp': expect.any(String), message: { - content: - 'This is a system message\n\nYou MUST respond in the users preferred language which is: English.', + content: EXPECTED_STORED_SYSTEM_MESSAGE, role: MessageRole.System, }, }, @@ -801,8 +804,7 @@ describe('Observability AI Assistant client', () => { '@timestamp': expect.any(String), message: { role: MessageRole.System, - content: - 'This is a system message\n\nYou MUST respond in the users preferred language which is: English.', + content: EXPECTED_STORED_SYSTEM_MESSAGE, }, }, { @@ -934,8 +936,7 @@ describe('Observability AI Assistant client', () => { { '@timestamp': expect.any(String), message: { - content: - 'This is a system message\n\nYou MUST respond in the users preferred language which is: English.', + content: EXPECTED_STORED_SYSTEM_MESSAGE, role: MessageRole.System, }, }, @@ -1307,7 +1308,7 @@ describe('Observability AI Assistant client', () => { async function requestAlertsFunctionCall() { const body = JSON.parse( (actionsClientMock.execute.mock.lastCall![0].params as any).subActionParams.body - ); + ) as OpenAI.ChatCompletionCreateParams; let nextLlmCallPromise: Promise; @@ -1326,7 +1327,7 @@ describe('Observability AI Assistant client', () => { await nextTick(); - for (let i = 0; i <= maxFunctionCalls; i++) { + for (let i = 0; i <= maxFunctionCalls + 1; i++) { await requestAlertsFunctionCall(); } @@ -1337,7 +1338,7 @@ describe('Observability AI Assistant client', () => { expect(functionClientMock.executeFunction).toHaveBeenCalledTimes(maxFunctionCalls); }); - it('does not give the LLM the choice to call a function anymore', () => { + it('asks the LLM to suggest next steps', () => { const firstBody = JSON.parse( (actionsClientMock.execute.mock.calls[0][0].params as any).subActionParams.body ); @@ -1345,7 +1346,7 @@ describe('Observability AI Assistant client', () => { (actionsClientMock.execute.mock.lastCall![0].params as any).subActionParams.body ); - expect(firstBody.tools.length).toBe(1); + expect(firstBody.tools.length).toEqual(1); expect(body.tools).toBeUndefined(); }); @@ -1546,7 +1547,7 @@ describe('Observability AI Assistant client', () => { await nextTick(); expect(chatSpy.mock.calls[0][1].messages[0].message.content).toEqual( - 'This is a system message\n\nYou MUST respond in the users preferred language which is: English.' + EXPECTED_STORED_SYSTEM_MESSAGE ); }); @@ -1576,7 +1577,7 @@ describe('Observability AI Assistant client', () => { await nextTick(); expect(chatSpy.mock.calls[0][1].messages[0].message.content).toEqual( - 'This is a system message\n\nYou MUST respond in the users preferred language which is: Orcish.' + EXPECTED_STORED_SYSTEM_MESSAGE.replace('English', 'Orcish') ); }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 0800cf816579a5..e4cb53be99754b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -14,11 +14,12 @@ import apm from 'elastic-apm-node'; import { decode, encode } from 'gpt-tokenizer'; import { findLastIndex, last, merge, noop, omit, pick, take } from 'lodash'; import { - catchError, filter, + identity, isObservable, last as lastOperator, lastValueFrom, + map, Observable, shareReplay, tap, @@ -26,16 +27,12 @@ import { } from 'rxjs'; import { Readable } from 'stream'; import { v4 } from 'uuid'; -import { withTokenBudget } from '../../../common/utils/with_token_budget'; -import { extendSystemMessage } from '../../../common/utils/extend_system_message'; import { ObservabilityAIAssistantConnectorType } from '../../../common/connectors'; import { ChatCompletionChunkEvent, ChatCompletionErrorEvent, createConversationNotFoundError, - createFunctionLimitExceededError, createTokenLimitReachedError, - isFunctionNotFoundError, MessageAddEvent, StreamingChatResponseEventType, TokenCountEvent, @@ -65,12 +62,15 @@ import { RecalledEntry, } from '../knowledge_base_service'; import type { ChatFn, ObservabilityAIAssistantResourceNames } from '../types'; +import { catchFunctionLimitExceededError } from '../util/catch_function_limit_exceeded_error'; import { getAccessQuery } from '../util/get_access_query'; +import { getSystemMessageFromInstructions } from '../util/get_system_message_from_instructions'; import { rejectTokenCountEvents } from '../util/reject_token_count_events'; +import { replaceSystemMessage } from '../util/replace_system_message'; import { createBedrockClaudeAdapter } from './adapters/bedrock/bedrock_claude_adapter'; +import { failOnNonExistingFunctionCall } from './adapters/fail_on_non_existing_function_call'; import { createOpenAiAdapter } from './adapters/openai_adapter'; import { LlmApiAdapter } from './adapters/types'; -import { failOnNonExistingFunctionCall } from './adapters/fail_on_non_existing_function_call'; export class ObservabilityAIAssistantClient { constructor( @@ -172,21 +172,32 @@ export class ObservabilityAIAssistantClient { kibanaPublicUrl, simulateFunctionCalling, isPublic = false, + instructions: requestInstructions = [], } = params; const isConversationUpdate = persist && !!params.conversationId; const conversationId = persist ? params.conversationId || v4() : ''; const title = params.title || ''; const responseLanguage = params.responseLanguage || 'English'; - const requestInstructions = params.instructions || []; + + const registeredInstructions = functionClient.getInstructions(); + + const knowledgeBaseInstructions: UserInstruction[] = []; + + if (responseLanguage) { + requestInstructions.push( + `You MUST respond in the users preferred language which is: ${responseLanguage}.` + ); + } + + let storedSystemMessage: string = ''; // will be set as soon as kb instructions are loaded if (persist && !isConversationUpdate && kibanaPublicUrl) { - const systemMessage = messages.find( - (message) => message.message.role === MessageRole.System + registeredInstructions.push( + `This conversation will be persisted in Kibana and available at this url: ${ + kibanaPublicUrl + `/app/observabilityAIAssistant/conversations/${conversationId}` + }.` ); - systemMessage!.message.content += `This conversation will be persisted in Kibana and available at this url: ${ - kibanaPublicUrl + `/app/observabilityAIAssistant/conversations/${conversationId}` - }.`; } const tokenCountResult = { @@ -281,17 +292,24 @@ export class ObservabilityAIAssistantClient { return await next(nextMessages.concat(contextFunctionRequest)); } else if (isUserMessage) { - const functions = - numFunctionsCalled >= MAX_FUNCTION_CALLS ? [] : allFunctions.concat(allActions); + const functionCallsExceeded = numFunctionsCalled > MAX_FUNCTION_CALLS; + const functions = functionCallsExceeded ? [] : allFunctions.concat(allActions); const spanName = lastMessage.message.name && lastMessage.message.name !== 'context' ? 'function_response' : 'user_message'; + const systemMessageForChatRequest = getSystemMessageFromInstructions({ + registeredInstructions, + requestInstructions, + knowledgeBaseInstructions, + availableFunctionNames: functions.map((fn) => fn.name) || [], + }); + const response$ = ( await chatWithTokenCountIncrement(spanName, { - messages: nextMessages, + messages: replaceSystemMessage(systemMessageForChatRequest, nextMessages), connectorId, signal, functions, @@ -299,12 +317,7 @@ export class ObservabilityAIAssistantClient { ).pipe( emitWithConcatenatedMessage(), shareReplay(), - catchError((error) => { - if (isFunctionNotFoundError(error) && functions.length === 0) { - throw createFunctionLimitExceededError(); - } - throw error; - }) + Boolean(functions.length) ? identity : catchFunctionLimitExceededError() ); response$.subscribe({ @@ -319,6 +332,18 @@ export class ObservabilityAIAssistantClient { (event): event is MessageAddEvent => event.type === StreamingChatResponseEventType.MessageAdd ), + // LLMs like to hallucinate parameters if the function does not define + // them, and it can lead to other hallicunations down the line + map((messageEvent) => { + const fnName = messageEvent.message.message.function_call?.name; + + if (fnName && !functions.find((fn) => fn.name === fnName)?.parameters) { + const clone = { ...messageEvent }; + clone.message.message.function_call!.arguments = ''; + return clone; + } + return messageEvent; + }), toArray() ) ); @@ -379,7 +404,7 @@ export class ObservabilityAIAssistantClient { chat: chatWithTokenCountIncrement, connectorId, name: functionCallName, - messages: nextMessages, + messages: replaceSystemMessage(storedSystemMessage, nextMessages), args: lastMessage.message.function_call!.arguments, signal, }) @@ -513,7 +538,7 @@ export class ObservabilityAIAssistantClient { omit(conversation._source, 'messages'), // update messages - { messages: nextMessages }, + { messages: replaceSystemMessage(storedSystemMessage, nextMessages) }, // update token count { @@ -528,6 +553,7 @@ export class ObservabilityAIAssistantClient { } ) ); + subscriber.next({ type: StreamingChatResponseEventType.ConversationUpdate, conversation: updatedConversation.conversation, @@ -545,7 +571,7 @@ export class ObservabilityAIAssistantClient { token_count: tokenCountResult, id: conversationId, }, - messages: nextMessages, + messages: replaceSystemMessage(storedSystemMessage, nextMessages), labels: {}, numeric_labels: {}, public: isPublic, @@ -560,14 +586,18 @@ export class ObservabilityAIAssistantClient { subscriber.complete(); }; - this.resolveInstructions(requestInstructions) - .then((instructions) => { - return next( - extendSystemMessage(messages, [ - `You MUST respond in the users preferred language which is: ${responseLanguage}.`, - instructions, - ]) - ); + this.fetchKnowledgeBaseInstructions() + .then((loadedKnowledgeBaseInstructions) => { + knowledgeBaseInstructions.push(...loadedKnowledgeBaseInstructions); + + storedSystemMessage = getSystemMessageFromInstructions({ + registeredInstructions, + requestInstructions, + knowledgeBaseInstructions, + availableFunctionNames: allFunctions.map((fn) => fn.name), + }); + + return next(messages); }) .catch((error) => { if (!signal.aborted) { @@ -972,30 +1002,12 @@ export class ObservabilityAIAssistantClient { return this.dependencies.knowledgeBaseService.deleteEntry({ id }); }; - private resolveInstructions = async (requestInstructions: Array) => { + fetchKnowledgeBaseInstructions = async () => { const knowledgeBaseInstructions = await this.dependencies.knowledgeBaseService.getInstructions( this.dependencies.namespace, this.dependencies.user ); - if (requestInstructions.length + knowledgeBaseInstructions.length === 0) { - return ''; - } - - const priorityInstructions = requestInstructions.map((instruction) => - typeof instruction === 'string' ? { doc_id: v4(), text: instruction } : instruction - ); - const overrideIds = priorityInstructions.map((instruction) => instruction.doc_id); - const instructions = priorityInstructions.concat( - knowledgeBaseInstructions.filter((instruction) => !overrideIds.includes(instruction.doc_id)) - ); - - const instructionsWithinBudget = withTokenBudget(instructions, 1000); - - const instructionsPrompt = `What follows is a set of instructions provided by the user, please abide by them as long as they don't conflict with anything you've been told so far:\n`; - - return `${instructionsPrompt}${instructionsWithinBudget - .map((instruction) => instruction.text) - .join('\n\n')}`; + return knowledgeBaseInstructions; }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts index 90cc6f3693e417..241ecd1350c68a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts @@ -13,7 +13,11 @@ import type { FunctionDefinition, FunctionResponse, } from '../../common/functions/types'; -import type { Message, ObservabilityAIAssistantScreenContextRequest } from '../../common/types'; +import type { + Message, + ObservabilityAIAssistantScreenContextRequest, + UserInstructionOrPlainText, +} from '../../common/types'; import type { ObservabilityAIAssistantRouteHandlerResources } from '../routes/types'; import { ChatFunctionClient } from './chat_function_client'; import type { ObservabilityAIAssistantClient } from './client'; @@ -43,6 +47,18 @@ export interface FunctionHandler { respond: RespondFunction; } +export type RegisteredInstruction = UserInstructionOrPlainText | RegisterInstructionCallback; + +type RegisterInstructionCallback = ({ + availableFunctionNames, +}: { + availableFunctionNames: string[]; +}) => UserInstructionOrPlainText | UserInstructionOrPlainText[] | undefined; + +export type RegisterInstruction = ( + ...instructions: Array +) => void; + export type RegisterFunction = < TParameters extends CompatibleJSONSchema = any, TResponse extends FunctionResponse = any, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/catch_function_limit_exceeded_error.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/catch_function_limit_exceeded_error.ts new file mode 100644 index 00000000000000..25eecc7e7723e7 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/catch_function_limit_exceeded_error.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { catchError, filter, of, OperatorFunction, shareReplay, throwError } from 'rxjs'; +import { + ChatCompletionChunkEvent, + MessageAddEvent, + MessageRole, + StreamingChatResponseEventType, +} from '../../../common'; +import { isFunctionNotFoundError } from '../../../common/conversation_complete'; +import { emitWithConcatenatedMessage } from '../../../common/utils/emit_with_concatenated_message'; + +export function catchFunctionLimitExceededError(): OperatorFunction< + ChatCompletionChunkEvent | MessageAddEvent, + ChatCompletionChunkEvent | MessageAddEvent +> { + return (source$) => { + const shared$ = source$.pipe(shareReplay()); + const chunksWithoutErrors$ = shared$.pipe( + catchError(() => of()), + shareReplay() + ); + + return shared$.pipe( + catchError((error) => { + if (isFunctionNotFoundError(error)) { + const withInjectedErrorMessage$ = chunksWithoutErrors$.pipe( + filter( + (msg): msg is ChatCompletionChunkEvent => + msg.type === StreamingChatResponseEventType.ChatCompletionChunk + ), + emitWithConcatenatedMessage(async (concatenatedMessage) => { + return { + ...concatenatedMessage, + message: { + ...concatenatedMessage.message, + content: `${concatenatedMessage.message.content}\n\n${i18n.translate( + 'xpack.observabilityAiAssistant.functionCallLimitExceeded', + { + defaultMessage: + '\n\nNote: the Assistant tried to call a function, even though the limit was exceeded', + } + )}`, + function_call: { + name: '', + arguments: '', + trigger: MessageRole.Assistant, + }, + }, + }; + }) + ); + + return withInjectedErrorMessage$; + } + return throwError(() => error); + }) + ); + }; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts new file mode 100644 index 00000000000000..9706871a63c625 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getSystemMessageFromInstructions } from './get_system_message_from_instructions'; + +describe('getSystemMessageFromInstructions', () => { + it('handles plain instructions', () => { + expect( + getSystemMessageFromInstructions({ + registeredInstructions: ['first', 'second'], + knowledgeBaseInstructions: [], + requestInstructions: [], + availableFunctionNames: [], + }) + ).toEqual(`first\n\nsecond`); + }); + + it('handles callbacks', () => { + expect( + getSystemMessageFromInstructions({ + registeredInstructions: [ + 'first', + ({ availableFunctionNames }) => { + return availableFunctionNames[0]; + }, + ], + knowledgeBaseInstructions: [], + requestInstructions: [], + availableFunctionNames: ['myFunction'], + }) + ).toEqual(`first\n\nmyFunction`); + }); + + it('overrides kb instructions with request instructions', () => { + expect( + getSystemMessageFromInstructions({ + registeredInstructions: ['first'], + knowledgeBaseInstructions: [{ doc_id: 'second', text: 'second_kb' }], + requestInstructions: [{ doc_id: 'second', text: 'second_request' }], + availableFunctionNames: [], + }) + ).toEqual( + `first\n\nWhat follows is a set of instructions provided by the user, please abide by them as long as they don't conflict with anything you've been told so far:\n\nsecond_request` + ); + }); + + it('includes kb instructions if there is no request instruction', () => { + expect( + getSystemMessageFromInstructions({ + registeredInstructions: ['first'], + knowledgeBaseInstructions: [{ doc_id: 'second', text: 'second_kb' }], + requestInstructions: [], + availableFunctionNames: [], + }) + ).toEqual( + `first\n\nWhat follows is a set of instructions provided by the user, please abide by them as long as they don't conflict with anything you've been told so far:\n\nsecond_kb` + ); + }); + + it('handles undefined values', () => { + expect( + getSystemMessageFromInstructions({ + registeredInstructions: [ + 'first', + ({ availableFunctionNames }) => { + return undefined; + }, + ], + knowledgeBaseInstructions: [], + requestInstructions: [], + availableFunctionNames: [], + }) + ).toEqual(`first`); + }); +}); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts new file mode 100644 index 00000000000000..4ea5aaecb67f90 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { compact } from 'lodash'; +import { v4 } from 'uuid'; +import { UserInstruction } from '../../../common/types'; +import { withTokenBudget } from '../../../common/utils/with_token_budget'; +import { RegisteredInstruction } from '../types'; + +export function getSystemMessageFromInstructions({ + registeredInstructions, + knowledgeBaseInstructions, + requestInstructions, + availableFunctionNames, +}: { + registeredInstructions: RegisteredInstruction[]; + knowledgeBaseInstructions: UserInstruction[]; + requestInstructions: Array; + availableFunctionNames: string[]; +}): string { + const allRegisteredInstructions = compact( + registeredInstructions.flatMap((instruction) => { + if (typeof instruction === 'function') { + return instruction({ availableFunctionNames }); + } + return instruction; + }) + ); + + const requestInstructionsWithId = requestInstructions.map((instruction) => + typeof instruction === 'string' ? { doc_id: v4(), text: instruction } : instruction + ); + + const requestOverrideIds = requestInstructionsWithId.map((instruction) => instruction.doc_id); + + // all request instructions, and those from the KB that are not defined as a request instruction + const allUserInstructions = requestInstructionsWithId.concat( + knowledgeBaseInstructions.filter( + (instruction) => !requestOverrideIds.includes(instruction.doc_id) + ) + ); + + const instructionsWithinBudget = withTokenBudget(allUserInstructions, 1000); + + return [ + ...allRegisteredInstructions, + ...(instructionsWithinBudget.length + ? [ + `What follows is a set of instructions provided by the user, please abide by them as long as they don't conflict with anything you've been told so far:`, + ...instructionsWithinBudget, + ] + : []), + ] + .map((instruction) => { + return typeof instruction === 'string' ? instruction : instruction.text; + }) + .join('\n\n'); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/replace_system_message.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/replace_system_message.ts new file mode 100644 index 00000000000000..c8c3b251c53e58 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/replace_system_message.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Message, MessageRole } from '../../../common'; + +export function replaceSystemMessage(systemMessage: string, messages: Message[]): Message[] { + return [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: systemMessage, + }, + }, + ...messages.filter((msg) => msg.message.role !== MessageRole.System), + ]; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_body.stories.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_body.stories.tsx index a6768198441da4..b556617726feff 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_body.stories.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_body.stories.tsx @@ -7,12 +7,10 @@ import { ComponentMeta, ComponentStoryObj } from '@storybook/react'; import React from 'react'; -import { - getAssistantSystemMessage, - MessageRole, -} from '@kbn/observability-ai-assistant-plugin/public'; +import { MessageRole } from '@kbn/observability-ai-assistant-plugin/public'; import { KibanaReactStorybookDecorator } from '../../utils/storybook_decorator'; import { ChatBody as Component } from './chat_body'; +import { buildSystemMessage } from '../../utils/builders'; const meta: ComponentMeta = { component: Component, @@ -25,7 +23,7 @@ const defaultProps: ComponentStoryObj = { args: { initialTitle: 'My Conversation', initialMessages: [ - getAssistantSystemMessage({ contexts: [] }), + buildSystemMessage(), { '@timestamp': new Date().toISOString(), message: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_flyout.stories.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_flyout.stories.tsx index 827c15ed44be25..edae8066986622 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_flyout.stories.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/chat/chat_flyout.stories.tsx @@ -7,7 +7,7 @@ import { ComponentStory } from '@storybook/react'; import React from 'react'; -import { getAssistantSystemMessage } from '@kbn/observability-ai-assistant-plugin/public'; +import { buildSystemMessage } from '../../utils/builders'; import { KibanaReactStorybookDecorator } from '../../utils/storybook_decorator'; import { ChatFlyout as Component } from './chat_flyout'; @@ -30,7 +30,7 @@ const Template: ComponentStory = (props: ChatFlyoutProps) => { const defaultProps: ChatFlyoutProps = { isOpen: true, initialTitle: 'How is this working', - initialMessages: [getAssistantSystemMessage({ contexts: [] })], + initialMessages: [buildSystemMessage()], onClose: () => {}, }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/builders.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/builders.ts index 57bdf17bd9a734..c92915897baf07 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/builders.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/builders.ts @@ -12,7 +12,6 @@ import { type Message, MessageRole, } from '@kbn/observability-ai-assistant-plugin/common'; -import { getAssistantSystemMessage } from '@kbn/observability-ai-assistant-plugin/public'; type BuildMessageProps = DeepPartial & { message: { @@ -104,7 +103,7 @@ export function buildFunctionResponseMessage( ); } -export function buildConversation(params?: Partial) { +export function buildConversation(params?: Partial): Conversation { return { '@timestamp': '', user: { @@ -115,10 +114,11 @@ export function buildConversation(params?: Partial) { title: '', last_updated: '', }, - messages: [getAssistantSystemMessage({ contexts: [] })], + messages: [buildSystemMessage()], labels: {}, numeric_labels: {}, namespace: '', + public: false, ...params, }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/create_mock_chat_service.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/create_mock_chat_service.ts index 07fde4462abb66..460722c49be64e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/create_mock_chat_service.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/utils/create_mock_chat_service.ts @@ -6,7 +6,10 @@ */ import type { DeeplyMockedKeys } from '@kbn/utility-types-jest'; -import type { ObservabilityAIAssistantChatService } from '@kbn/observability-ai-assistant-plugin/public'; +import { + MessageRole, + ObservabilityAIAssistantChatService, +} from '@kbn/observability-ai-assistant-plugin/public'; type MockedChatService = DeeplyMockedKeys; @@ -15,11 +18,17 @@ export const createMockChatService = (): MockedChatService => { chat: jest.fn(), complete: jest.fn(), sendAnalyticsEvent: jest.fn(), - getContexts: jest.fn().mockReturnValue([{ name: 'core', description: '' }]), getFunctions: jest.fn().mockReturnValue([]), hasFunction: jest.fn().mockReturnValue(false), hasRenderFunction: jest.fn().mockReturnValue(true), renderFunction: jest.fn(), + getSystemMessage: jest.fn().mockReturnValue({ + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.System, + content: '', + }, + }), }; return mockChatService; }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts index f287c26e6cd83d..7b62ca4f5a6d2b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/alerts.ts @@ -47,7 +47,6 @@ export function registerAlertsFunction({ functions.registerFunction( { name: 'alerts', - contexts: ['core'], description: 'Get alerts for Observability. Display the response in tabular format if appropriate.', descriptionForUser: 'Get alerts for Observability', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts index c86fe66a4c6e2a..89ebfa90cb774c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts @@ -28,7 +28,6 @@ export function registerChangesFunction({ { name: 'changes', description: 'Returns change points like spikes and dips for logs and metrics.', - contexts: ['core'], parameters: changesFunctionParameters, }, async ({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts index e9490a725a6408..dd21e8d082eb9c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts @@ -108,5 +108,24 @@ describe('correctCommonEsqlMistakes', () => { | WHERE statement LIKE "SELECT%" | STATS avg_duration = AVG(duration)` ); + + expectQuery( + `FROM metrics-apm* + | WHERE metricset.name == "service_destination" AND @timestamp > NOW() - 24 hours + | EVAL total_events = span.destination.service.response_time.count + | EVAL total_latency = span.destination.service.response_time.sum.us + | EVAL is_failure = CASE(event.outcome == "failure", 1, 0) + | STATS + avg_throughput = AVG(total_events), + avg_latency_per_request = AVG(total_latency / total_events), + failure_rate = AVG(is_failure) + BY span.destination.service.resource`, + `FROM metrics-apm* + | WHERE metricset.name == "service_destination" AND @timestamp > NOW() - 24 hours + | EVAL total_events = span.destination.service.response_time.count + | EVAL total_latency = span.destination.service.response_time.sum.us + | EVAL is_failure = CASE(event.outcome == "failure", 1, 0) + | STATS avg_throughput = AVG(total_events), avg_latency_per_request = AVG(total_latency / total_events), failure_rate = AVG(is_failure) BY span.destination.service.resource` + ); }); }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts index 8b3f18359ce093..01d6e67fe217a0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts @@ -104,7 +104,7 @@ function isValidColumnName(column: string) { } function escapeColumns(line: string) { - const [, command, body] = line.match(/^([A-Za-z_]+)(.*)$/) ?? ['', '', '']; + const [, command, body] = line.match(/^([A-Za-z_]+)(.*)$/s) ?? ['', '', '']; const escapedBody = split(body.trim(), ',') .map((statement) => { @@ -198,7 +198,7 @@ function escapeExpressionsInSort(sortCommand: string) { export function correctCommonEsqlMistakes(content: string, log: Logger) { return content.replaceAll(/```esql\n(.*?)\n```/gms, (_, query: string) => { - const commands = splitIntoCommands(query); + const commands = splitIntoCommands(query.trim()); const formattedCommands: string[] = commands.map(({ name, command }, index) => { let formattedCommand = command; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_query_with_actions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_query_with_actions.ts index 213b7e967970ad..15b050c3a38970 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_query_with_actions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_query_with_actions.ts @@ -22,6 +22,7 @@ const fixedQueryByOneAction = async (queryString: string) => { const correctText = firstAction.edits[0].text; const problematicString = queryString.substring(range.startColumn - 1, range.endColumn - 1); const fixedQuery = queryString.replace(problematicString, correctText); + return { query: fixedQuery, shouldRunAgain: Boolean(actions.length), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts index 7c33b119340aa8..2cf8600d9db2ff 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts @@ -23,9 +23,10 @@ import { } from '@kbn/observability-ai-assistant-plugin/common/utils/concatenate_chat_completion_chunks'; import { emitWithConcatenatedMessage } from '@kbn/observability-ai-assistant-plugin/common/utils/emit_with_concatenated_message'; import { createFunctionResponseMessage } from '@kbn/observability-ai-assistant-plugin/common/utils/create_function_response_message'; +import { ESQLSearchReponse } from '@kbn/es-types'; import type { FunctionRegistrationParameters } from '..'; import { correctCommonEsqlMistakes } from './correct_common_esql_mistakes'; -import { correctQueryWithActions } from './correct_query_with_actions'; +import { validateEsqlQuery } from './validate_esql_query'; const readFile = promisify(Fs.readFile); const readdir = promisify(Fs.readdir); @@ -68,15 +69,30 @@ const loadEsqlDocs = once(async () => { ); }); -export function registerQueryFunction({ - client, - functions, - resources, -}: FunctionRegistrationParameters) { +export function registerQueryFunction({ functions, resources }: FunctionRegistrationParameters) { + functions.registerInstruction(({ availableFunctionNames }) => + availableFunctionNames.includes('query') + ? `You MUST use the "query" function when the user wants to: + - visualize data + - run any arbitrary query + - breakdown or filter ES|QL queries that are displayed on the current page + - convert queries from another language to ES|QL + - asks general questions about ES|QL + + DO NOT UNDER ANY CIRCUMSTANCES generate ES|QL queries or explain anything about the ES|QL query language yourself. + DO NOT UNDER ANY CIRCUMSTANCES try to correct an ES|QL query yourself - always use the "query" function for this. + + Even if the "context" function was used before that, follow it up with the "query" function. If a query fails, do not attempt to correct it yourself. Again you should call the "query" function, + even if it has been called before. + + When the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt. + If the "execute_query" function has been called, summarize these results for the user. The user does not see a visualization in this case.` + : undefined + ); + functions.registerFunction( { name: 'execute_query', - contexts: ['core'], visibility: FunctionVisibility.UserOnly, description: 'Display the results of an ES|QL query. ONLY use this if the "query" function has been used before or if the user or screen context has provided a query you can use.', @@ -91,25 +107,39 @@ export function registerQueryFunction({ } as const, }, async ({ arguments: { query } }) => { - const response = await ( - await resources.context.core - ).elasticsearch.client.asCurrentUser.transport.request({ + const client = (await resources.context.core).elasticsearch.client.asCurrentUser; + const { error, errorMessages } = await validateEsqlQuery({ + query, + client, + }); + + if (!!error) { + return { + content: { + message: 'The query failed to execute', + error, + errorMessages, + }, + }; + } + const response = (await client.transport.request({ method: 'POST', path: '_query', body: { query, version: ESQL_LATEST_VERSION, }, - }); + })) as ESQLSearchReponse; - return { content: response }; + return { + content: response, + }; } ); functions.registerFunction( { name: 'query', - contexts: ['core'], - description: `This function generates, executes and/or visualizes a query based on the user's request. It also explains how ES|QL works and how to convert queries from one language to another. Make sure you call one of the get_dataset functions first if you need index or field names. This function takes no arguments.`, + description: `This function generates, executes and/or visualizes a query based on the user's request. It also explains how ES|QL works and how to convert queries from one language to another. Make sure you call one of the get_dataset functions first if you need index or field names. This function takes no input.`, visibility: FunctionVisibility.AssistantOnly, }, async ({ messages, connectorId, chat }, signal) => { @@ -140,10 +170,16 @@ export function registerQueryFunction({ Extract data? Request \`DISSECT\` AND \`GROK\`. Convert a column based on a set of conditionals? Request \`EVAL\` and \`CASE\`. + ONLY use ${VisualizeESQLUserIntention.executeAndReturnResults} if you are absolutely sure + it is executable. If one of the get_dataset_info functions were not called before, OR if + one of the get_dataset_info functions returned no data, opt for an explanation only and + mention that there is no data for these indices. You can still use + ${VisualizeESQLUserIntention.generateQueryOnly} and generate an example ES|QL query. + For determining the intention of the user, the following options are available: ${VisualizeESQLUserIntention.generateQueryOnly}: the user only wants to generate the query, - but not run it. + but not run it, or they ask a general question about ES|QL. ${VisualizeESQLUserIntention.executeAndReturnResults}: the user wants to execute the query, and have the assistant return/analyze/summarize the results. they don't need a @@ -356,10 +392,9 @@ export function registerQueryFunction({ if (msg.message.function_call.name) { return msg; } - let esqlQuery = correctCommonEsqlMistakes(msg.message.content, resources.logger).match( - /```esql([\s\S]*?)```/ - )?.[1]; - esqlQuery = await correctQueryWithActions(esqlQuery ?? ''); + const esqlQuery = correctCommonEsqlMistakes(msg.message.content, resources.logger) + .match(/```esql([\s\S]*?)```/)?.[1] + ?.trim(); let functionCall: ConcatenatedMessage['message']['function_call'] | undefined; @@ -401,6 +436,7 @@ export function registerQueryFunction({ name: 'query', content: {}, data: { + // add the included docs for debugging documentation: { intention: args.intention, keywords, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts new file mode 100644 index 00000000000000..dafba4352634ef --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/validate_esql_query.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { validateQuery } from '@kbn/esql-validation-autocomplete'; +import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ESQL_LATEST_VERSION } from '@kbn/esql-utils'; +import { ESQLSearchReponse } from '@kbn/es-types'; +import { esFieldTypeToKibanaFieldType, type KBN_FIELD_TYPES } from '@kbn/field-types'; + +export async function validateEsqlQuery({ + query, + client, +}: { + query: string; + client: ElasticsearchClient; +}): Promise<{ + columns?: Array<{ + id: string; + name: string; + meta: { + type: KBN_FIELD_TYPES; + }; + }>; + error?: Error; + errorMessages?: string[]; +}> { + const { errors } = await validateQuery(query, getAstAndSyntaxErrors, { + // setting this to true, we don't want to validate the index / fields existence + ignoreOnMissingCallbacks: true, + }); + + const errorMessages = errors?.map((error) => { + return 'text' in error ? error.text : error.message; + }); + + // With limit 0 I get only the columns, it is much more performant + const performantQuery = `${query} | limit 0`; + + return client.transport + .request({ + method: 'POST', + path: '_query', + body: { + query: performantQuery, + version: ESQL_LATEST_VERSION, + }, + }) + .then((res) => { + const esqlResponse = res as ESQLSearchReponse; + + const columns = + esqlResponse.columns?.map(({ name, type }) => ({ + id: name, + name, + meta: { type: esFieldTypeToKibanaFieldType(type) }, + })) ?? []; + + return { columns }; + }) + .catch((error) => { + return { + error, + errorMessages, + }; + }); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts index e8b2320b917c13..1a7d64c0d324ff 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/visualize_esql.ts @@ -4,14 +4,10 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { esFieldTypeToKibanaFieldType } from '@kbn/field-types'; -import type { ESQLSearchReponse } from '@kbn/es-types'; -import { validateQuery } from '@kbn/esql-validation-autocomplete'; -import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; -import { ESQL_LATEST_VERSION } from '@kbn/esql-utils'; import { VisualizeESQLUserIntention } from '@kbn/observability-ai-assistant-plugin/common/functions/visualize_esql'; import { visualizeESQLFunction } from '../../common/functions/visualize_esql'; import { FunctionRegistrationParameters } from '.'; +import { validateEsqlQuery } from './query/validate_esql_query'; const getMessageForLLM = ( intention: VisualizeESQLUserIntention, @@ -34,37 +30,12 @@ export function registerVisualizeESQLFunction({ functions.registerFunction( visualizeESQLFunction, async ({ arguments: { query, intention }, connectorId, messages }, signal) => { - // recomputing the errors here as the user might click the Visualize query button - // and call the function manually. - const { errors } = await validateQuery(query, getAstAndSyntaxErrors, { - // setting this to true, we don't want to validate the index / fields existence - ignoreOnMissingCallbacks: true, + const { columns, errorMessages } = await validateEsqlQuery({ + query, + client: (await resources.context.core).elasticsearch.client.asCurrentUser, }); - const errorMessages = errors?.map((error) => { - return 'text' in error ? error.text : error.message; - }); - // With limit 0 I get only the columns, it is much more performant - const performantQuery = `${query} | limit 0`; - const coreContext = await resources.context.core; - - const response = (await ( - await coreContext - ).elasticsearch.client.asCurrentUser.transport.request({ - method: 'POST', - path: '_query', - body: { - query: performantQuery, - version: ESQL_LATEST_VERSION, - }, - })) as ESQLSearchReponse; - const columns = - response.columns?.map(({ name, type }) => ({ - id: name, - name, - meta: { type: esFieldTypeToKibanaFieldType(type) }, - })) ?? []; - const message = getMessageForLLM(intention, query, Boolean(errorMessages.length)); + const message = getMessageForLLM(intention, query, Boolean(errorMessages?.length)); return { data: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts index 7990f353062da1..190ce8c9ef95c1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts @@ -82,7 +82,8 @@ describe('observabilityAIAssistant rule_connector', () => { service: { getClient: async () => ({ complete: completeMock }), getFunctionClient: async () => ({ - getContexts: () => [{ name: 'core', description: 'my_system_message' }], + getFunctions: () => [], + getInstructions: () => [], }), }, context: { @@ -119,6 +120,7 @@ describe('observabilityAIAssistant rule_connector', () => { expect(result).toEqual({ actionId: 'observability-ai-assistant', status: 'ok' }); expect(initResources).toHaveBeenCalledTimes(1); expect(completeMock).toHaveBeenCalledTimes(1); + expect(completeMock).toHaveBeenCalledWith( expect.objectContaining({ persist: true, @@ -130,7 +132,7 @@ describe('observabilityAIAssistant rule_connector', () => { '@timestamp': expect.any(String), message: { role: MessageRole.System, - content: 'my_system_message', + content: '', }, }, { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts index 567c326945ef82..b46fec93d1dd1b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts @@ -32,6 +32,7 @@ import { } from '@kbn/observability-ai-assistant-plugin/common'; import { concatenateChatCompletionChunks } from '@kbn/observability-ai-assistant-plugin/common/utils/concatenate_chat_completion_chunks'; import { CompatibleJSONSchema } from '@kbn/observability-ai-assistant-plugin/common/functions/types'; +import { getSystemMessageFromInstructions } from '@kbn/observability-ai-assistant-plugin/server/service/util/get_system_message_from_instructions'; import { convertSchemaToOpenApi } from './convert_schema_to_open_api'; import { OBSERVABILITY_AI_ASSISTANT_CONNECTOR_ID } from '../../common/rule_connector'; @@ -171,9 +172,6 @@ async function executor( }); }); - const systemMessage = functionClient - .getContexts() - .find((def) => def.name === 'core')?.description; const backgroundInstruction = getBackgroundProcessInstruction( execOptions.params.rule, execOptions.params.alerts @@ -193,7 +191,12 @@ async function executor( '@timestamp': new Date().toISOString(), message: { role: MessageRole.System, - content: systemMessage, + content: getSystemMessageFromInstructions({ + availableFunctionNames: functionClient.getFunctions().map((fn) => fn.definition.name), + registeredInstructions: functionClient.getInstructions(), + knowledgeBaseInstructions: [], + requestInstructions: [], + }), }, }, { From 9d5abba33898de51b41d3b9397db05e66b13f16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=BCrkan=20YALAMAN?= Date: Wed, 24 Apr 2024 11:35:36 +0200 Subject: [PATCH 002/138] [Search] Move Attach Index to top (#181446) ## Summary https://github.com/elastic/kibana/assets/1410658/de571e71-88c1-4576-94ee-55763ba8af98 Screenshot 2024-04-23 at 16 27 52 ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../connector_configuration.tsx | 533 +++++++++--------- .../native_connector_configuration.tsx | 191 ++++--- 2 files changed, 350 insertions(+), 374 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/connector_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/connector_configuration.tsx index 09814ebab8b7c6..3f05c4d305110f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/connector_configuration.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/connector_configuration.tsx @@ -96,307 +96,278 @@ export const ConnectorConfiguration: React.FC = () => { - - ) : ( - i18n.translate( - 'xpack.enterpriseSearch.content.connectorDetail.configuration.apiKey.noApiKeyLabel', - { - defaultMessage: - 'Before you can generate an API key, you need to attach an index. Scroll to the bottom of this page for instructions.', - } - ) - ), - status: hasApiKey ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.generateApiKey.title', + { + <> + + + + } + {connector.index_name && ( + <> + + - - - - {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.configurationFileLink', - { defaultMessage: 'example config file' } - )} - - ), - }} - /> - - - - {getConnectorTemplate({ - apiKeyData, - connectorData: { - id: connector.id, - service_type: connector.service_type, - }, - host: cloudContext.elasticsearchUrl, - })} - - - - - {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.deploymentModeLink', - { defaultMessage: 'documentation' } - )} - - ), - }} + children: ( + - - - ), - status: - !connector.status || connector.status === ConnectorStatus.CREATED - ? 'incomplete' - : 'complete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.deployConnector.title', + ), + status: hasApiKey ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.generateApiKey.title', + { + defaultMessage: 'Generate an API key', + } + ), + titleSize: 'xs', + }, { - defaultMessage: 'Set up and deploy connector', - } - ), - titleSize: 'xs', - }, - { - children: ( - - updateConnectorConfiguration({ - configuration, - connectorId: connector.id, - }) - } - subscriptionLink={docLinks.licenseManagement} - stackManagementLink={http.basePath.prepend( - '/app/management/stack/license_management' - )} - > - {!connector.status || connector.status === ConnectorStatus.CREATED ? ( - - {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.waitingForConnectorText', - { - defaultMessage: - 'Your connector has not connected to Search. Troubleshoot your configuration and refresh the page.', - } - )} - - fetchConnector({ connectorId: connector.id })} - isLoading={isLoading} - > - {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.waitingForConnector.button.label', - { - defaultMessage: 'Recheck now', - } - )} - - - ) : ( - - )} - - {connector.status && hasAdvancedFilteringFeature && !!advancedSnippet && ( - + + + + {i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.configurationFileLink', + { defaultMessage: 'example config file' } + )} + + ), + }} + /> + + + + {getConnectorTemplate({ + apiKeyData, + connectorData: { + id: connector.id, + service_type: connector.service_type, + }, + host: cloudContext.elasticsearchUrl, + })} + + + + + {i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.deploymentModeLink', + { defaultMessage: 'documentation' } + )} + + ), + }} + /> + + + ), + status: + !connector.status || connector.status === ConnectorStatus.CREATED + ? 'incomplete' + : 'complete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.deployConnector.title', + { + defaultMessage: 'Set up and deploy connector', + } + ), + titleSize: 'xs', + }, + { + children: ( + + updateConnectorConfiguration({ + configuration, + connectorId: connector.id, + }) + } + subscriptionLink={docLinks.licenseManagement} + stackManagementLink={http.basePath.prepend( + '/app/management/stack/license_management' )} - iconType="iInCircle" - color="warning" > - - {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.advancedSyncRulesDocs', - { defaultMessage: 'Advanced Sync Rules' } - )} - - ), - }} - /> - - )} - - ), - status: - connector.status === ConnectorStatus.CONNECTED ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.enhance.title', - { - defaultMessage: 'Configure your connector', - } - ), - titleSize: 'xs', - }, - { - children: ( - - {!connector.index_name && ( - - - - + {!connector.status || connector.status === ConnectorStatus.CREATED ? ( + {i18n.translate( - 'xpack.enterpriseSearch.content.connectors.configuration.connectorNoIndexCallOut.description', + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.waitingForConnectorText', { defaultMessage: - "You won't be able to start syncing content until your connector is attached to an index.", + 'Your connector has not connected to Search. Troubleshoot your configuration and refresh the page.', } )} - - - - - )} - - - {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.scheduleSync.description', - { - defaultMessage: - 'Finalize your connector by triggering a one-time sync, or setting a recurring sync to keep your data source in sync over time', - } + + fetchConnector({ connectorId: connector.id })} + isLoading={isLoading} + > + {i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.waitingForConnector.button.label', + { + defaultMessage: 'Recheck now', + } + )} + + + ) : ( + )} - - - - - - + {connector.status && hasAdvancedFilteringFeature && !!advancedSnippet && ( + + + {i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.connectorPackage.advancedSyncRulesDocs', + { defaultMessage: 'Advanced Sync Rules' } + )} + + ), + }} + /> + + )} + + ), + status: + connector.status === ConnectorStatus.CONNECTED ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.enhance.title', + { + defaultMessage: 'Configure your connector', + } + ), + titleSize: 'xs', + }, + { + children: ( + + + {i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.schedule.button.label', + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.scheduleSync.description', { - defaultMessage: 'Set schedule and sync', + defaultMessage: + 'Finalize your connector by triggering a one-time sync, or setting a recurring sync to keep your data source in sync over time', } )} - + - - + + + + + {i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.schedule.button.label', + { + defaultMessage: 'Set schedule and sync', + } + )} + + + + + + - - - ), - status: connector.scheduling.full.enabled ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.schedule.title', - { - defaultMessage: 'Sync your data', - } - ), - titleSize: 'xs', - }, - ]} - /> + ), + status: connector.scheduling.full.enabled ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.connector_detail.configurationConnector.steps.schedule.title', + { + defaultMessage: 'Sync your data', + } + ), + titleSize: 'xs', + }, + ]} + /> + + )} - { - <> - - - - } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/native_connector_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/native_connector_configuration.tsx index cc89fc84c2432e..4660ad75fe9054 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/native_connector_configuration.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/native_connector_configuration.tsx @@ -142,106 +142,111 @@ export const NativeConnectorConfiguration: React.FC = () => { )} - , - status: hasResearched ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.researchConfigurationTitle', + { + <> + + + + } + {connector.index_name && ( + <> + + - ), - status: hasConfigured ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.configurationTitle', + children: , + status: hasResearched ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.researchConfigurationTitle', + { + defaultMessage: 'Research configuration requirements', + } + ), + titleSize: 'xs', + }, { - defaultMessage: 'Configuration', - } - ), - titleSize: 'xs', - }, - { - children: ( - - ), - status: hasApiKey ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.manageApiKeyTitle', + children: ( + + ), + status: hasConfigured ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.configurationTitle', + { + defaultMessage: 'Configuration', + } + ), + titleSize: 'xs', + }, { - defaultMessage: 'Manage API key', - } - ), - titleSize: 'xs', - }, - { - children: ( - - - - - - - - - - - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnectorAdvancedConfiguration.schedulingButtonLabel', - { - defaultMessage: 'Set schedule and sync', - } - )} - + children: ( + + ), + status: hasApiKey ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.manageApiKeyTitle', + { + defaultMessage: 'Manage API key', + } + ), + titleSize: 'xs', + }, + { + children: ( + + + + + - - + + + + + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnectorAdvancedConfiguration.schedulingButtonLabel', + { + defaultMessage: 'Set schedule and sync', + } + )} + + + + + + - - - ), - status: hasConfiguredAdvanced ? 'complete' : 'incomplete', - title: i18n.translate( - 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.advancedConfigurationTitle', - { - defaultMessage: 'Sync your data', - } - ), - titleSize: 'xs', - }, - ]} - /> + ), + status: hasConfiguredAdvanced ? 'complete' : 'incomplete', + title: i18n.translate( + 'xpack.enterpriseSearch.content.indices.configurationConnector.nativeConnector.steps.advancedConfigurationTitle', + { + defaultMessage: 'Sync your data', + } + ), + titleSize: 'xs', + }, + ]} + /> + + )} - { - <> - - - - } From 42e6f0cd73f9d80944136925f844757a0d775dc2 Mon Sep 17 00:00:00 2001 From: Angela Chuang <6295984+angorayc@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:41:21 +0100 Subject: [PATCH 003/138] [SecuritySolution] Add "exist filter" when value count is filtered in Lens cell action (#181151) ## Summary Original issue and steps to reproduce: https://github.com/elastic/kibana/issues/181120 Before: https://github.com/elastic/kibana/assets/6295984/10c7a2ce-d814-4750-8481-8f05b55384f8 After: https://github.com/elastic/kibana/assets/6295984/00dfbcb6-244b-4f2b-8dd4-a1f7435385cf ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Sergi Massaneda --- .../src/actions/filter/add_filter.ts | 41 ++++ .../src/actions/filter/create_filter.ts | 5 +- .../src/actions/filter/filter_in.ts | 6 +- .../src/actions/filter/filter_out.ts | 7 +- .../src/actions/filter/index.ts | 1 + packages/kbn-cell-actions/src/index.ts | 1 + .../actions/filter/lens/create_action.test.ts | 190 ++++++++++++++++++ .../actions/filter/lens/create_action.ts | 40 +++- 8 files changed, 270 insertions(+), 21 deletions(-) create mode 100644 packages/kbn-cell-actions/src/actions/filter/add_filter.ts create mode 100644 x-pack/plugins/security_solution/public/actions/filter/lens/create_action.test.ts diff --git a/packages/kbn-cell-actions/src/actions/filter/add_filter.ts b/packages/kbn-cell-actions/src/actions/filter/add_filter.ts new file mode 100644 index 00000000000000..484a2bd9792cbb --- /dev/null +++ b/packages/kbn-cell-actions/src/actions/filter/add_filter.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { FilterManager } from '@kbn/data-plugin/public'; +import type { DefaultActionsSupportedValue } from '../types'; +import { createExistsFilter, createFilter } from './create_filter'; + +interface AddFilterParams { + filterManager: FilterManager; + key: string; + value: DefaultActionsSupportedValue; + negate: boolean; + dataViewId?: string; +} + +export const addFilter = ({ filterManager, key, value, negate, dataViewId }: AddFilterParams) => { + filterManager.addFilters(createFilter({ key, value, negate, dataViewId })); +}; + +interface AddExistsFilterParams { + filterManager: FilterManager; + key: string; + negate: boolean; + dataViewId?: string; +} +export const addExistsFilter = ({ + filterManager, + key, + negate, + dataViewId, +}: AddExistsFilterParams) => { + filterManager.addFilters(createExistsFilter({ key, negate, dataViewId })); +}; + +export const isEmptyFilterValue = (value: Array) => + value.length === 0 || value.every((v) => v === ''); diff --git a/packages/kbn-cell-actions/src/actions/filter/create_filter.ts b/packages/kbn-cell-actions/src/actions/filter/create_filter.ts index fe10ab7df43d6b..a34786ebb693a4 100644 --- a/packages/kbn-cell-actions/src/actions/filter/create_filter.ts +++ b/packages/kbn-cell-actions/src/actions/filter/create_filter.ts @@ -15,10 +15,7 @@ import { } from '@kbn/es-query'; import { DefaultActionsSupportedValue } from '../types'; -export const isEmptyFilterValue = (value: Array) => - value.length === 0 || value.every((v) => v === ''); - -const createExistsFilter = ({ +export const createExistsFilter = ({ key, negate, dataViewId, diff --git a/packages/kbn-cell-actions/src/actions/filter/filter_in.ts b/packages/kbn-cell-actions/src/actions/filter/filter_in.ts index 8dd4487bfd9443..9e88b97208ba77 100644 --- a/packages/kbn-cell-actions/src/actions/filter/filter_in.ts +++ b/packages/kbn-cell-actions/src/actions/filter/filter_in.ts @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import type { FilterManager, KBN_FIELD_TYPES } from '@kbn/data-plugin/public'; import { NotificationsStart } from '@kbn/core-notifications-browser'; -import { createFilter, isEmptyFilterValue } from './create_filter'; +import { addFilter, isEmptyFilterValue } from './add_filter'; import { FILTER_CELL_ACTION_TYPE } from '../../constants'; import { createCellActionFactory } from '../factory'; import { @@ -77,12 +77,12 @@ export const addFilterIn = ({ dataViewId?: string; }) => { if (filterManager != null) { - const filter = createFilter({ + addFilter({ + filterManager, key: fieldName, value, negate: isEmptyFilterValue(value), dataViewId, }); - filterManager.addFilters(filter); } }; diff --git a/packages/kbn-cell-actions/src/actions/filter/filter_out.ts b/packages/kbn-cell-actions/src/actions/filter/filter_out.ts index e20d7c267b910a..56cea2541143e7 100644 --- a/packages/kbn-cell-actions/src/actions/filter/filter_out.ts +++ b/packages/kbn-cell-actions/src/actions/filter/filter_out.ts @@ -8,7 +8,8 @@ import { i18n } from '@kbn/i18n'; import type { FilterManager, KBN_FIELD_TYPES } from '@kbn/data-plugin/public'; import { NotificationsStart } from '@kbn/core-notifications-browser'; -import { createFilter, isEmptyFilterValue } from './create_filter'; +import { addFilter, isEmptyFilterValue } from './add_filter'; + import { FILTER_CELL_ACTION_TYPE } from '../../constants'; import { createCellActionFactory } from '../factory'; import { @@ -81,12 +82,12 @@ export const addFilterOut = ({ dataViewId?: string; }) => { if (filterManager != null) { - const filter = createFilter({ + addFilter({ + filterManager, key: fieldName, value, negate: !isEmptyFilterValue(value), dataViewId, }); - filterManager.addFilters(filter); } }; diff --git a/packages/kbn-cell-actions/src/actions/filter/index.ts b/packages/kbn-cell-actions/src/actions/filter/index.ts index 19a32c05db6cbe..61908d4700dce5 100644 --- a/packages/kbn-cell-actions/src/actions/filter/index.ts +++ b/packages/kbn-cell-actions/src/actions/filter/index.ts @@ -8,3 +8,4 @@ export { createFilterInActionFactory, addFilterIn } from './filter_in'; export { createFilterOutActionFactory, addFilterOut } from './filter_out'; +export { addExistsFilter } from './add_filter'; diff --git a/packages/kbn-cell-actions/src/index.ts b/packages/kbn-cell-actions/src/index.ts index 4e478baec441fa..4b7dde4cdcaf1f 100644 --- a/packages/kbn-cell-actions/src/index.ts +++ b/packages/kbn-cell-actions/src/index.ts @@ -34,6 +34,7 @@ export { createFilterOutActionFactory, addFilterIn, addFilterOut, + addExistsFilter, } from './actions/filter'; // Action factory diff --git a/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.test.ts b/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.test.ts new file mode 100644 index 00000000000000..1efbbc99601466 --- /dev/null +++ b/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.test.ts @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { addExistsFilter, addFilterIn, addFilterOut } from '@kbn/cell-actions'; +import { of } from 'rxjs'; +import type { CellValueContext } from '@kbn/embeddable-plugin/public'; +import type { CreateFilterLensActionParams } from './create_action'; +import { createFilterLensAction } from './create_action'; +import type { Trigger } from '@kbn/ui-actions-plugin/public'; + +jest.mock('@kbn/cell-actions', () => ({ + addFilterIn: jest.fn(), + addFilterOut: jest.fn(), + addExistsFilter: jest.fn(), +})); + +jest.mock('../../../timelines/store', () => ({ + timelineSelectors: { + getTimelineByIdSelector: jest.fn().mockReturnValue(() => ({})), + }, +})); + +describe('createFilterLensAction', () => { + const mockServices = { + timelineFilterManager: 'mockTimelineFilterManager', + data: { query: { filterManager: 'mockFilterManager' } }, + application: { currentAppId$: of('appId') }, + topValuesPopover: { + closePopover: jest.fn(), + }, + notifications: { + toasts: { + addWarning: jest.fn(), + }, + }, + }; + const mockStore = { + getState: jest.fn(), + }; + + const mockUserCountData = [ + { + columnMeta: { + field: 'user.count', + sourceParams: { + type: 'value_count', + indexPatternId: 'indexPatternId', + }, + }, + value: [1], + }, + ] as unknown as CellValueContext['data']; + + const mockUserNameData = [ + { + columnMeta: { + field: 'user.name', + sourceParams: { + type: 'string', + indexPatternId: 'indexPatternId', + }, + }, + value: 'elastic', + }, + ] as unknown as CellValueContext['data']; + + const mockTrigger = { + id: 'triggerId', + title: 'triggerTitle', + description: 'triggerDescription', + } as Trigger; + + const params = { + id: 'embeddable_filterIn', + order: 0, + store: mockStore, + services: mockServices, + } as unknown as CreateFilterLensActionParams; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should create a "filter In" action with the field value', async () => { + const { execute } = createFilterLensAction(params); + await execute({ + data: mockUserNameData, + trigger: mockTrigger, + }); + expect(addFilterIn).toHaveBeenCalledWith({ + filterManager: 'mockFilterManager', + fieldName: 'user.name', + value: ['elastic'], + dataViewId: 'indexPatternId', + }); + expect(addFilterOut).not.toHaveBeenCalled(); + }); + + it('should create a "filter Out" action with the field value', async () => { + const testParams = { + ...params, + id: 'embeddable_filterOut', + negate: true, + }; + const { execute } = createFilterLensAction(testParams); + await execute({ + data: mockUserNameData, + trigger: mockTrigger, + }); + expect(addFilterIn).not.toHaveBeenCalled(); + expect(addFilterOut).toHaveBeenCalledWith({ + filterManager: 'mockFilterManager', + fieldName: 'user.name', + value: ['elastic'], + dataViewId: 'indexPatternId', + }); + }); + + it('should create an "exists" filter when value type equals "value_count"', async () => { + const { execute } = createFilterLensAction(params); + await execute({ + data: mockUserCountData, + trigger: mockTrigger, + }); + expect(addExistsFilter).toHaveBeenCalledWith({ + filterManager: 'mockFilterManager', + key: 'user.count', + negate: false, + dataViewId: 'indexPatternId', + }); + }); + + it('should create an "Not exists" filter when value type equals "value_count"', async () => { + const testParams = { + ...params, + negate: true, + }; + const { execute } = createFilterLensAction(testParams); + await execute({ + data: mockUserCountData, + trigger: mockTrigger, + }); + expect(addExistsFilter).toHaveBeenCalledWith({ + filterManager: 'mockFilterManager', + key: 'user.count', + negate: true, + dataViewId: 'indexPatternId', + }); + expect(addFilterIn).not.toHaveBeenCalled(); + }); + + it('should show a warning toast when the value is not supported', async () => { + const { execute } = createFilterLensAction(params); + await execute({ + data: [ + { + columnMeta: { + field: 'user.name', + sourceParams: { + type: 'string', + indexPatternId: 'indexPatternId', + }, + }, + value: [[1], '1', 'foo'], + }, + ] as unknown as CellValueContext['data'], + trigger: mockTrigger, + }); + expect(mockServices.notifications.toasts.addWarning).toHaveBeenCalled(); + }); + + it('should not create a filter when the field is missing', async () => { + const { execute } = createFilterLensAction(params); + await execute({ + data: [ + { + columnMeta: {}, + value: 'elastic', + }, + ] as unknown as CellValueContext['data'], + trigger: mockTrigger, + }); + expect(addFilterIn).not.toHaveBeenCalled(); + expect(addFilterOut).not.toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.ts b/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.ts index e0da4447506cfc..ecfe71bc3a1125 100644 --- a/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.ts +++ b/x-pack/plugins/security_solution/public/actions/filter/lens/create_action.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { addFilterIn, addFilterOut } from '@kbn/cell-actions'; +import { addExistsFilter, addFilterIn, addFilterOut } from '@kbn/cell-actions'; import { isValueSupportedByDefaultActions, valueToArray, @@ -31,19 +31,21 @@ function isDataColumnsValid(data?: CellValueContext['data']): boolean { ); } +export interface CreateFilterLensActionParams { + id: string; + order: number; + store: SecurityAppStore; + services: StartServices; + negate?: boolean; +} + export const createFilterLensAction = ({ id, order, store, services, negate, -}: { - id: string; - order: number; - store: SecurityAppStore; - services: StartServices; - negate?: boolean; -}) => { +}: CreateFilterLensActionParams) => { const { application, notifications, data: dataService, topValuesPopover } = services; let currentAppId: string | undefined; @@ -72,6 +74,7 @@ export const createFilterLensAction = ({ isInSecurityApp(currentAppId), execute: async ({ data }) => { const field = data[0]?.columnMeta?.field; + const isCounter = data[0]?.columnMeta?.sourceParams?.type === 'value_count'; const rawValue = data[0]?.value; const mayBeDataViewId = data[0]?.columnMeta?.sourceParams?.indexPatternId; const dataViewId = typeof mayBeDataViewId === 'string' ? mayBeDataViewId : undefined; @@ -87,15 +90,30 @@ export const createFilterLensAction = ({ topValuesPopover.closePopover(); - const addFilter = negate === true ? addFilterOut : addFilterIn; - const timeline = getTimelineById(store.getState(), TimelineId.active); // timeline is open add the filter to timeline, otherwise add filter to global filters const filterManager = timeline?.show ? services.timelineFilterManager : dataService.query.filterManager; - addFilter({ filterManager, fieldName: field, value, dataViewId }); + // If value type is value_count, we want to filter an `Exists` filter instead of a `Term` filter + if (isCounter) { + addExistsFilter({ + filterManager, + key: field, + negate: !!negate, + dataViewId, + }); + return; + } + + const addFilter = negate === true ? addFilterOut : addFilterIn; + addFilter({ + filterManager, + fieldName: field, + value, + dataViewId, + }); }, }); }; From 214833af0bc45dad269529a1904cfbb934da945e Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 24 Apr 2024 11:51:58 +0200 Subject: [PATCH 004/138] [Obs AI Assistant] Hide unavailable connectors (#181455) Hides unavailable connectors (e.g. due to license mismatches). --- .../server/routes/connectors/route.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/connectors/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/connectors/route.ts index 79134b9fef8d06..24d63d3f7fa06b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/connectors/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/connectors/route.ts @@ -20,9 +20,24 @@ const listConnectorsRoute = createObservabilityAIAssistantServerRoute({ await plugins.actions.start() ).getActionsClientWithRequest(request); - const connectors = await actionsClient.getAll(); + const [availableTypes, connectors] = await Promise.all([ + actionsClient + .listTypes({ + includeSystemActionTypes: false, + }) + .then((types) => + types + .filter((type) => type.enabled && type.enabledInLicense && type.enabledInConfig) + .map((type) => type.id) + ), + actionsClient.getAll(), + ]); - return connectors.filter((connector) => isSupportedConnectorType(connector.actionTypeId)); + return connectors.filter( + (connector) => + availableTypes.includes(connector.actionTypeId) && + isSupportedConnectorType(connector.actionTypeId) + ); }, }); From c17ba8cd1078a2885fd8baa7ab8c9faaf6e39671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:08:26 +0100 Subject: [PATCH 005/138] [Profiling] Adding "estimated value" label on sample columns (#181449) The service samples and transaction samples are estimated values, so I'm adding a tooltip to make it clear for users. Screenshot 2024-04-23 at 15 51 44 --- .../apm_transactions.tsx | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/apm_transactions.tsx b/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/apm_transactions.tsx index 9d5986deca9b21..54ce9ebc9eddb1 100644 --- a/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/apm_transactions.tsx +++ b/x-pack/plugins/observability_solution/profiling/public/components/frame_information_window/apm_transactions.tsx @@ -12,7 +12,9 @@ import { EuiFieldSearch, EuiFlexGroup, EuiFlexItem, + EuiIcon, EuiLink, + EuiToolTip, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { isEmpty } from 'lodash'; @@ -78,6 +80,22 @@ const findServicesAndTransactions = ( }; }; +function EstimatedLabel({ label }: { label: string }) { + return ( + + <> + {label} + + + ); +} + +const SAMPLES_COLUMN_WIDTH = '152px'; + export function APMTransactions({ functionName, serviceNames }: Props) { const { query: { rangeFrom, rangeTo }, @@ -199,11 +217,15 @@ export function APMTransactions({ functionName, serviceNames }: Props) { }, { field: 'serviceSamples', - name: i18n.translate('xpack.profiling.apmTransactions.columns.serviceSamplesName', { - defaultMessage: 'Service Samples', - }), - width: '150px', + width: SAMPLES_COLUMN_WIDTH, sortable: true, + name: ( + + ), render(_, { serviceSamples }) { return asNumber(serviceSamples); }, @@ -239,10 +261,14 @@ export function APMTransactions({ functionName, serviceNames }: Props) { }, { field: 'transactionSamples', - name: i18n.translate('xpack.profiling.apmTransactions.columns.transactionSamples', { - defaultMessage: 'Transaction Samples', - }), - width: '150px', + name: ( + + ), + width: SAMPLES_COLUMN_WIDTH, render(_, { transactionSamples }) { if (isLoadingTransactions) { return '--'; From 67a2eb54c98678b546b08bf6f810f44bd31e1055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 24 Apr 2024 12:36:41 +0200 Subject: [PATCH 006/138] [Obs AI Assistant] Hide insight components when license is incorrect or there are no configured connectors (#181519) Closes https://github.com/elastic/kibana/issues/181435 --- .../public/components/insight/insight.tsx | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx index fb06d77efee8b3..6c96287a281322 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight.tsx @@ -16,8 +16,10 @@ import { EuiCallOut, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { cloneDeep, isArray, last, once } from 'lodash'; +import { cloneDeep, isArray, isEmpty, last, once } from 'lodash'; import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import useObservable from 'react-use/lib/useObservable'; +import { ILicense } from '@kbn/licensing-plugin/public'; import { MessageRole, type Message } from '../../../common/types'; import { ObservabilityAIAssistantChatServiceContext } from '../../context/observability_ai_assistant_chat_service_context'; import { useAbortableAsync } from '../../hooks/use_abortable_async'; @@ -290,9 +292,20 @@ export function Insight({ }; const { - services: { http }, + services: { + http, + plugins: { + start: { licensing }, + }, + }, } = useKibana(); + const license = useObservable(licensing.license$); + const hasEnterpriseLicense = license?.hasAtLeast('enterprise'); + if (isEmpty(connectors.connectors) || !hasEnterpriseLicense) { + return null; + } + let children: React.ReactNode = null; if ( From e1ec9ee17e89e1299778cc639f7f4ea085471861 Mon Sep 17 00:00:00 2001 From: Abdul Wahab Zahid Date: Wed, 24 Apr 2024 13:20:31 +0200 Subject: [PATCH 007/138] [Dataset quality] Flyout Summary section (#179479) Closes https://github.com/elastic/kibana/issues/170492 ## Summary The PR Implements the Dataset Quality Flyout summary KPIs: "Docs count", "Size", "Services", "Hosts" and "Degraded docs". |Stateful|Serverless| |:---:|:---| |Screenshot 2024-03-26 at 19 14 34|Screenshot 2024-03-26 at 17 02 05| "Show all" links for "Services" and "Hosts" metrics depend on some development in APM and Infra plugins and therefore will be implemented in a follow up issue. Note that "Size" metric is excluded on Serverless as the endpoint uses ES's `_stats` endpoint which is not available on Serverless (at the time of creation of this PR). The code contains some conditions and cases to tackle this, which should be considered as a temporary measure. All of the code related to these changes should either be removed or modified once there's a way to calculate the size of an index/dataStream on Serverless (see [related ](https://github.com/elastic/kibana/issues/178954)). The following changes are made in particular: - Size UI component on the Flyout will be hidden on Serverless - `dataset_quality/data_streams/{dataStream}/details` endpoint will return `NaN` for size on Serverless - Unit, integration and end-to-end tests on Serverless handle "sizeBytes" property accordingly --- .github/CODEOWNERS | 1 + .../dataset_quality/common/api_types.ts | 14 +- .../dataset_quality/common/constants.ts | 7 + .../common/data_streams_stats/types.ts | 13 +- .../dataset_quality/common/translations.ts | 27 ++ .../summary_panel/estimated_data.tsx | 5 +- .../dataset_quality/table/columns.tsx | 5 +- .../table/degraded_docs_percentage_link.tsx | 3 +- .../components/flyout/dataset_summary.tsx | 19 +- .../degraded_docs_trend/degraded_docs.tsx | 166 ++----- .../degraded_docs_chart.tsx | 51 +- .../degraded_docs_trend/lens_attributes.ts | 2 +- .../public/components/flyout/fields_list.tsx | 13 +- .../public/components/flyout/flyout.tsx | 71 ++- .../flyout/flyout_summary/flyout_summary.tsx | 104 +++++ .../flyout_summary/flyout_summary_header.tsx | 78 ++++ .../flyout_summary_kpi_item.tsx | 100 ++++ .../flyout_summary/flyout_summary_kpis.tsx | 77 +++ .../flyout_summary/get_summary_kpis.test.ts | 156 +++++++ .../flyout/flyout_summary/get_summary_kpis.ts | 137 ++++++ .../flyout/integration_actions_menu.tsx | 24 +- .../components/flyout/integration_summary.tsx | 14 +- .../hooks/use_dataset_quality_flyout.tsx | 27 +- .../hooks/use_dataset_quality_table.tsx | 8 +- .../public/hooks/use_degraded_docs_chart.tsx | 58 ++- .../public/hooks/use_link_to_logs_explorer.ts | 3 + .../data_stream_details_client.ts | 30 +- .../services/data_stream_details/types.ts | 3 + .../src/notifications.ts | 9 + .../src/state_machine.ts | 118 +++-- .../dataset_quality_controller/src/types.ts | 18 +- .../dataset_quality/public/types.ts | 5 +- .../get_data_stream_details.test.ts | 439 +++++++++++------- .../get_data_stream_settings.test.ts | 227 +++++++++ .../get_data_stream_details/index.ts | 149 +++++- .../server/routes/data_streams/routes.ts | 52 ++- .../dataset_quality/tsconfig.json | 4 +- .../data_streams/data_stream_details.spec.ts | 26 +- .../data_streams/data_stream_settings.spec.ts | 106 +++++ .../utils/data_stream.ts | 17 +- .../apps/dataset_quality/data/logs_data.ts | 8 +- .../dataset_quality/dataset_quality_flyout.ts | 133 ++++++ .../dataset_quality_summary.ts | 2 +- .../page_objects/dataset_quality.ts | 61 ++- .../common/reporting/generate_csv_discover.ts | 4 +- .../test_suites/observability/config.ts | 5 +- .../common/dataset_quality_api_supertest.ts | 129 +++++ .../common/services.ts | 38 ++ .../data_stream_details.ts | 96 ++++ .../data_stream_settings.ts | 101 ++++ .../dataset_quality_api_integration/index.ts | 14 + .../utils/data_stream.ts | 26 ++ .../utils/expect_to_reject.ts | 17 + .../utils/index.ts | 9 + .../test_suites/observability/index.ts | 1 + .../dataset_quality/data/logs_data.ts | 6 +- .../dataset_quality/dataset_quality_flyout.ts | 150 +++++- .../dataset_quality_summary.ts | 4 +- x-pack/test_serverless/tsconfig.json | 1 + 59 files changed, 2725 insertions(+), 466 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_header.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpi_item.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpis.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.test.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_settings.test.ts create mode 100644 x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/dataset_quality_api_supertest.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/services.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_details.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/expect_to_reject.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/index.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4a6c4b24a68003..0c57ef0d7dc9a0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1133,6 +1133,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql # Logs /x-pack/test/api_integration/apis/logs_ui @elastic/obs-ux-logs-team /x-pack/test/dataset_quality_api_integration @elastic/obs-ux-logs-team +/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration @elastic/obs-ux-logs-team /x-pack/test/functional/apps/observability_logs_explorer @elastic/obs-ux-logs-team /x-pack/test_serverless/functional/test_suites/observability/observability_logs_explorer @elastic/obs-ux-logs-team /x-pack/test/functional/apps/dataset_quality @elastic/obs-ux-logs-team diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index ae098c08c8ec8c..4dac346e2a26a8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -76,9 +76,19 @@ export const degradedDocsRt = rt.type({ export type DegradedDocs = rt.TypeOf; +export const dataStreamSettingsRt = rt.partial({ + createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless +}); + +export type DataStreamSettings = rt.TypeOf; + export const dataStreamDetailsRt = rt.partial({ - createdOn: rt.number, lastActivity: rt.number, + degradedDocsCount: rt.number, + docsCount: rt.number, + sizeBytes: rt.union([rt.null, rt.number]), // rt.null is only needed for https://github.com/elastic/kibana/issues/178954 + services: rt.record(rt.string, rt.array(rt.string)), + hosts: rt.record(rt.string, rt.array(rt.string)), }); export type DataStreamDetails = rt.TypeOf; @@ -95,6 +105,8 @@ export const getDataStreamsDegradedDocsStatsResponseRt = rt.exact( }) ); +export const getDataStreamsSettingsResponseRt = rt.exact(dataStreamSettingsRt); + export const getDataStreamsDetailsResponseRt = rt.exact(dataStreamDetailsRt); export const dataStreamsEstimatedDataInBytesRT = rt.type({ diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index 325cae3dc99b3f..a5be06438d3c8c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -18,4 +18,11 @@ export const DEFAULT_SORT_DIRECTION = 'asc'; export const NONE = 'none'; export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' }; +export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false }; + export const DEFAULT_DEGRADED_DOCS = { percentage: 0, count: 0 }; + +export const NUMBER_FORMAT = '0,0.[000]'; +export const BYTE_NUMBER_FORMAT = '0.0 b'; + +export const MAX_HOSTS_METRIC_VALUE = 50; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 7fbbf4d8f3e90e..66fbffd452dc63 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -30,8 +30,17 @@ export type GetDataStreamsDegradedDocsStatsResponse = export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[]; export type DegradedDocsStatType = GetDataStreamsDegradedDocsStatsResponse['degradedDocs'][0]; -export type GetDataStreamDetailsParams = +export type GetDataStreamSettingsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/settings`>['params']['path']; +export type GetDataStreamSettingsResponse = + APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/settings`>; + +type GetDataStreamDetailsPathParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>['params']['path']; +type GetDataStreamDetailsQueryParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>['params']['query']; +export type GetDataStreamDetailsParams = GetDataStreamDetailsPathParams & + GetDataStreamDetailsQueryParams; export type GetDataStreamDetailsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>; @@ -47,4 +56,4 @@ export type GetIntegrationDashboardsResponse = export type DashboardType = GetIntegrationDashboardsResponse['dashboards'][0]; export type { DataStreamStat } from './data_stream_stat'; -export type { DataStreamDetails } from '../api_types'; +export type { DataStreamDetails, DataStreamSettings } from '../api_types'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index c2441a648566fe..d31f3d537eb221 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -88,6 +88,10 @@ export const flyoutIntegrationNameText = i18n.translate( } ); +export const flyoutSummaryText = i18n.translate('xpack.datasetQuality.flyoutSummaryTitle', { + defaultMessage: 'Summary', +}); + export const flyoutDegradedDocsText = i18n.translate( 'xpack.datasetQuality.flyout.degradedDocsTitle', { @@ -110,6 +114,29 @@ export const flyoutDegradedDocsPercentageText = i18n.translate( } ); +export const flyoutDocsCountTotalText = i18n.translate( + 'xpack.datasetQuality.flyoutDocsCountTotal', + { + defaultMessage: 'Docs count (total)', + } +); + +export const flyoutSizeText = i18n.translate('xpack.datasetQuality.flyoutSizeText', { + defaultMessage: 'Size', +}); + +export const flyoutServicesText = i18n.translate('xpack.datasetQuality.flyoutServicesText', { + defaultMessage: 'Services', +}); + +export const flyoutHostsText = i18n.translate('xpack.datasetQuality.flyoutHostsText', { + defaultMessage: 'Hosts', +}); + +export const flyoutShowAllText = i18n.translate('xpack.datasetQuality.flyoutShowAllText', { + defaultMessage: 'Show all', +}); + /* Summary Panel */ diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/estimated_data.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/estimated_data.tsx index 803cd6ab50a6a2..f44bd783169160 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/estimated_data.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/estimated_data.tsx @@ -6,9 +6,10 @@ */ import React from 'react'; +import { formatNumber } from '@elastic/eui'; -import { formatBytes } from '@kbn/formatters'; import { useSummaryPanelContext } from '../../../hooks'; +import { BYTE_NUMBER_FORMAT } from '../../../../common/constants'; import { summaryPanelEstimatedDataText, summaryPanelEstimatedDataTooltipText, @@ -22,7 +23,7 @@ export function EstimatedData() { ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 1fb229e1b7155d..1558e2ce506150 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -16,6 +16,7 @@ import { EuiToolTip, EuiButtonIcon, EuiText, + formatNumber, EuiSkeletonRectangle, } from '@elastic/eui'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; @@ -24,10 +25,10 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import React from 'react'; import { css } from '@emotion/react'; -import { formatBytes } from '@kbn/formatters'; import { DEGRADED_QUALITY_MINIMUM_PERCENTAGE, POOR_QUALITY_MINIMUM_PERCENTAGE, + BYTE_NUMBER_FORMAT, } from '../../../../common/constants'; import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; import { QualityIndicator } from '../../quality_indicator'; @@ -207,7 +208,7 @@ export const getDatasetQualityTableColumns = ({ borderRadius="m" isLoading={loadingDataStreamStats} > - {formatBytes(dataStreamStat.sizeBytes || 0)} + {formatNumber(dataStreamStat.sizeBytes || 0, BYTE_NUMBER_FORMAT)} ), width: '100px', diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx index 70f33c5a134bcf..d8aade6d74e51b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/degraded_docs_percentage_link.tsx @@ -7,6 +7,7 @@ import { EuiSkeletonRectangle, EuiFlexGroup, EuiLink } from '@elastic/eui'; import React from 'react'; +import { _IGNORED } from '../../../../common/es_fields'; import { useLinkToLogsExplorer } from '../../../hooks'; import { QualityPercentageIndicator } from '../../quality_indicator'; import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; @@ -24,7 +25,7 @@ export const DegradedDocsPercentageLink = ({ const logsExplorerLinkProps = useLinkToLogsExplorer({ dataStreamStat, - query: { language: 'kuery', query: '_ignored:*' }, + query: { language: 'kuery', query: `${_IGNORED}: *` }, }); return ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/dataset_summary.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/dataset_summary.tsx index a0954ab44e49b9..052f4b63f0da60 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/dataset_summary.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/dataset_summary.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; -import { DataStreamDetails } from '../../../common/data_streams_stats'; +import { DataStreamDetails, DataStreamSettings } from '../../../common/data_streams_stats'; import { flyoutDatasetCreatedOnText, flyoutDatasetDetailsText, @@ -18,18 +18,27 @@ import { FieldsList, FieldsListLoading } from './fields_list'; interface DatasetSummaryProps { fieldFormats: FieldFormatsStart; + dataStreamSettings?: DataStreamSettings; + dataStreamSettingsLoading: boolean; dataStreamDetails?: DataStreamDetails; + dataStreamDetailsLoading: boolean; } -export function DatasetSummary({ dataStreamDetails, fieldFormats }: DatasetSummaryProps) { +export function DatasetSummary({ + dataStreamSettings, + dataStreamSettingsLoading, + dataStreamDetails, + dataStreamDetailsLoading, + fieldFormats, +}: DatasetSummaryProps) { const dataFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ ES_FIELD_TYPES.DATE, ]); const formattedLastActivity = dataStreamDetails?.lastActivity ? dataFormatter.convert(dataStreamDetails?.lastActivity) : '-'; - const formattedCreatedOn = dataStreamDetails?.createdOn - ? dataFormatter.convert(dataStreamDetails.createdOn) + const formattedCreatedOn = dataStreamSettings?.createdOn + ? dataFormatter.convert(dataStreamSettings.createdOn) : '-'; return ( @@ -39,10 +48,12 @@ export function DatasetSummary({ dataStreamDetails, fieldFormats }: DatasetSumma { fieldTitle: flyoutDatasetLastActivityText, fieldValue: formattedLastActivity, + isLoading: dataStreamDetailsLoading, }, { fieldTitle: flyoutDatasetCreatedOnText, fieldValue: formattedCreatedOn, + isLoading: dataStreamSettingsLoading, }, ]} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs.tsx index 9a5348ed1495da..89087a26a19601 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs.tsx @@ -5,166 +5,94 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { css } from '@emotion/react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiFlexGroup, + EuiFlexItem, EuiPanel, EuiSpacer, EuiTitle, - EuiText, - EuiSuperDatePicker, - OnRefreshProps, EuiToolTip, EuiIcon, EuiCode, + OnTimeChangeProps, EuiSkeletonRectangle, } from '@elastic/eui'; -import { - UnifiedBreakdownFieldSelector, - fieldSupportsBreakdown, -} from '@kbn/unified-histogram-plugin/public'; -import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public'; +import type { DataViewField } from '@kbn/data-views-plugin/common'; +import { useDegradedDocsChart } from '../../../hooks'; -import { useCreateDataView } from '../../../hooks'; -import { indexNameToDataStreamParts } from '../../../../common/utils'; -import { DEFAULT_LOGS_DATA_VIEW, DEFAULT_TIME_RANGE } from '../../../../common/constants'; +import { DEFAULT_TIME_RANGE, DEFAULT_DATEPICKER_REFRESH } from '../../../../common/constants'; import { flyoutDegradedDocsText } from '../../../../common/translations'; import { TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; -import { useDatasetQualityContext } from '../../dataset_quality/context'; import { DegradedDocsChart } from './degraded_docs_chart'; -const DEFAULT_REFRESH = { value: 60000, pause: false }; - export function DegradedDocs({ dataStream, - timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_REFRESH }, - breakdownField, + timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH }, + lastReloadTime, + onTimeRangeChange, }: { dataStream?: string; timeRange?: TimeRangeConfig; - breakdownField?: string; + lastReloadTime: number; + onTimeRangeChange: (props: Pick) => void; }) { - const { service } = useDatasetQualityContext(); - const { dataView } = useCreateDataView({ - indexPatternString: getDataViewIndexPattern(dataStream), - }); + const { dataView, breakdown, ...chartProps } = useDegradedDocsChart({ dataStream }); const [breakdownDataViewField, setBreakdownDataViewField] = useState( undefined ); - const [lastReloadTime, setLastReloadTime] = useState(Date.now()); useEffect(() => { - if (dataView) { - const dataViewField = getDataViewField(dataView, breakdownField); - if (dataViewField) { - const isFieldBreakable = fieldSupportsBreakdown(dataViewField); - if (isFieldBreakable) { - setBreakdownDataViewField(dataViewField); - } else { - setBreakdownDataViewField(undefined); - // TODO: If needed, notify user that the field is not breakable - } - } else { - setBreakdownDataViewField(undefined); - } + if (breakdown.dataViewField && breakdown.fieldSupportsBreakdown) { + setBreakdownDataViewField(breakdown.dataViewField); + } else { + setBreakdownDataViewField(undefined); } - }, [dataView, breakdownField]); - - const handleRefresh = useCallback((_refreshProps: OnRefreshProps) => { - setLastReloadTime(Date.now()); - }, []); - - const handleTimeChange = useCallback( - (durationRange) => { - service.send({ - type: 'UPDATE_INSIGHTS_TIME_RANGE', - timeRange: { - from: durationRange.start, - to: durationRange.end, - refresh: timeRange.refresh ?? DEFAULT_REFRESH, - }, - }); - }, - [service, timeRange.refresh] - ); - const handleBreakdownFieldChange = useCallback( - (field: DataViewField | undefined) => { - service.send({ - type: 'BREAKDOWN_FIELD_CHANGE', - breakdownField: field?.name ?? null, - }); - }, - [service] - ); + if (breakdown.dataViewField && !breakdown.fieldSupportsBreakdown) { + // TODO: If needed, notify user that the field is not breakable + } + }, [setBreakdownDataViewField, breakdown.dataViewField, breakdown.fieldSupportsBreakdown]); return ( - - + - - {flyoutDegradedDocsText} + +
{flyoutDegradedDocsText}
 - + - - {dataView ? ( - - ) : ( - - )} - - - - - + + + - + + + ); @@ -183,13 +111,3 @@ const degradedDocsTooltip = ( }} /> ); - -function getDataViewIndexPattern(dataStream: string | undefined) { - return dataStream ? `${indexNameToDataStreamParts(dataStream).type}-*-*` : DEFAULT_LOGS_DATA_VIEW; -} - -function getDataViewField(dataView: DataView | undefined, fieldName: string | undefined) { - return fieldName && dataView - ? dataView.fields.find((field) => field.name === fieldName) - : undefined; -} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs_chart.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs_chart.tsx index 7a861308ba1ee8..9d13d0e3b26bab 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs_chart.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs_chart.tsx @@ -5,12 +5,11 @@ * 2.0. */ -import React from 'react'; +import React, { useCallback } from 'react'; import { css } from '@emotion/react'; -import { EuiFlexGroup, EuiLoadingChart } from '@elastic/eui'; +import { EuiFlexGroup, EuiLoadingChart, OnTimeChangeProps } from '@elastic/eui'; import { ViewMode } from '@kbn/embeddable-plugin/common'; import { KibanaErrorBoundary } from '@kbn/shared-ux-error-boundary'; -import { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { flyoutDegradedDocsTrendText } from '../../../../common/translations'; import { TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; @@ -25,25 +24,35 @@ const DISABLED_ACTIONS = [ 'create-ml-ad-job-action', ]; +interface DegradedDocsChartProps + extends Pick< + ReturnType, + 'attributes' | 'isChartLoading' | 'onChartLoading' | 'extraActions' + > { + timeRange: TimeRangeConfig; + lastReloadTime: number; + onTimeRangeChange: (props: Pick) => void; +} + export function DegradedDocsChart({ - dataStream, + attributes, + isChartLoading, + onChartLoading, + extraActions, timeRange, lastReloadTime, - dataView, - breakdownDataViewField, -}: { - dataStream?: string; - timeRange: TimeRangeConfig; - lastReloadTime: number; - dataView?: DataView; - breakdownDataViewField?: DataViewField; -}) { + onTimeRangeChange, +}: DegradedDocsChartProps) { const { services: { lens }, } = useKibanaContextForPlugin(); - const { attributes, filterQuery, extraActions, isChartLoading, handleChartLoading } = - useDegradedDocsChart({ dataStream, breakdownDataViewField }); + const handleBrushEnd = useCallback( + ({ range: [start, end] }: { range: number[] }) => { + onTimeRangeChange({ start: new Date(start).toISOString(), end: new Date(end).toISOString() }); + }, + [onTimeRangeChange] + ); return ( <> @@ -59,8 +68,11 @@ export function DegradedDocsChart({ ) : ( )} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/lens_attributes.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/lens_attributes.ts index 0f106769d427bc..c731c8f3cb77e7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/lens_attributes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/lens_attributes.ts @@ -242,7 +242,7 @@ function getChartColumns(breakdownField?: string): Record; + fields: Array<{ fieldTitle: string; fieldValue: ReactNode; isLoading: boolean }>; actionsMenu?: ReactNode; }) { return ( @@ -36,7 +37,7 @@ export function FieldsList({ - {fields.map(({ fieldTitle, fieldValue }, index) => ( + {fields.map(({ fieldTitle, fieldValue, isLoading: isFieldLoading }, index) => ( @@ -44,9 +45,11 @@ export function FieldsList({ {fieldTitle} - - {fieldValue} - + + + {fieldValue} + + {index < fields.length - 1 ? : null} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx index 4960dd73ccc409..56caa187414772 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx @@ -5,6 +5,8 @@ * 2.0. */ +import React, { Fragment } from 'react'; +import { css } from '@emotion/react'; import { EuiButtonEmpty, EuiFlexGroup, @@ -13,26 +15,27 @@ import { EuiFlyoutBody, EuiFlyoutFooter, EuiSpacer, + EuiHorizontalRule, + EuiPanel, } from '@elastic/eui'; -import React, { Fragment } from 'react'; import { flyoutCancelText } from '../../../common/translations'; import { useDatasetQualityFlyout } from '../../hooks'; import { DatasetSummary, DatasetSummaryLoading } from './dataset_summary'; import { Header } from './header'; import { IntegrationSummary } from './integration_summary'; import { FlyoutProps } from './types'; -import { DegradedDocs } from './degraded_docs_trend/degraded_docs'; +import { FlyoutSummary } from './flyout_summary/flyout_summary'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function Flyout({ dataset, closeFlyout }: FlyoutProps) { const { dataStreamStat, + dataStreamSettings, dataStreamDetails, - dataStreamDetailsLoading, fieldFormats, timeRange, - breakdownField, + loadingState, } = useDatasetQualityFlyout(); return ( @@ -45,26 +48,44 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) { > <>
- - + + + + - + - {dataStreamDetailsLoading ? ( - - ) : dataStreamStat ? ( - - - - {dataStreamStat.integration && ( - - )} - - ) : null} + + {loadingState.dataStreamDetailsLoading && loadingState.dataStreamSettingsLoading ? ( + + ) : dataStreamStat ? ( + + + + {dataStreamStat.integration && ( + <> + + + + )} + + ) : null} + @@ -85,3 +106,9 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) { ); } + +const flyoutBodyStyles = css` + .euiFlyoutBody__overflowContent { + padding: 0; + } +`; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx new file mode 100644 index 00000000000000..7ba5f315e607f1 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useState } from 'react'; +import { OnRefreshProps, OnTimeChangeProps, EuiSpacer } from '@elastic/eui'; + +import { DegradedDocs } from '../degraded_docs_trend/degraded_docs'; +import { DataStreamDetails } from '../../../../common/api_types'; +import { DEFAULT_TIME_RANGE, DEFAULT_DATEPICKER_REFRESH } from '../../../../common/constants'; +import { useDatasetQualityContext } from '../../dataset_quality/context'; +import { FlyoutDataset, TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; +import { FlyoutSummaryHeader } from './flyout_summary_header'; +import { FlyoutSummaryKpis, FlyoutSummaryKpisLoading } from './flyout_summary_kpis'; + +export function FlyoutSummary({ + dataStream, + dataStreamStat, + dataStreamDetails, + dataStreamDetailsLoading, + timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH }, +}: { + dataStream: string; + dataStreamStat?: FlyoutDataset; + dataStreamDetails?: DataStreamDetails; + dataStreamDetailsLoading: boolean; + timeRange?: TimeRangeConfig; +}) { + const { service } = useDatasetQualityContext(); + const [lastReloadTime, setLastReloadTime] = useState(Date.now()); + + const updateTimeRange = useCallback( + ({ start, end, refreshInterval }: OnRefreshProps) => { + service.send({ + type: 'UPDATE_INSIGHTS_TIME_RANGE', + timeRange: { + from: start, + to: end, + refresh: { ...DEFAULT_DATEPICKER_REFRESH, value: refreshInterval }, + }, + }); + }, + [service] + ); + + const handleTimeChange = useCallback( + ({ isInvalid, ...timeRangeProps }: OnTimeChangeProps) => { + if (!isInvalid) { + updateTimeRange({ refreshInterval: timeRange.refresh.value, ...timeRangeProps }); + } + }, + [updateTimeRange, timeRange.refresh] + ); + + const handleTimeRangeChange = useCallback( + ({ start, end }: Pick) => { + updateTimeRange({ start, end, refreshInterval: timeRange.refresh.value }); + }, + [updateTimeRange, timeRange.refresh] + ); + + const handleRefresh = useCallback( + (refreshProps: OnRefreshProps) => { + updateTimeRange(refreshProps); + setLastReloadTime(Date.now()); + }, + [updateTimeRange] + ); + + return ( + <> + + + + + {dataStreamStat ? ( + + ) : ( + + )} + + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_header.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_header.tsx new file mode 100644 index 00000000000000..518f4ef4b654e8 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_header.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { css } from '@emotion/react'; +import { + EuiFlexGroup, + EuiIcon, + EuiSuperDatePicker, + EuiTitle, + EuiToolTip, + OnRefreshProps, + OnTimeChangeProps, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { flyoutSummaryText } from '../../../../common/translations'; +import { TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; + +export function FlyoutSummaryHeader({ + timeRange, + onTimeChange, + onRefresh, +}: { + timeRange: TimeRangeConfig; + onTimeChange: (timeChangeProps: OnTimeChangeProps) => void; + onRefresh: (refreshProps: OnRefreshProps) => void; +}) { + return ( + + + + {flyoutSummaryText} + + + + + + + + + + + ); +} + +const flyoutSummaryTooltip = ( + +); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpi_item.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpi_item.tsx new file mode 100644 index 00000000000000..358f0eaaacbb53 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpi_item.tsx @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiTitle, + EuiText, + EuiLink, + useEuiTheme, + EuiSkeletonTitle, + EuiSkeletonRectangle, +} from '@elastic/eui'; + +export function FlyoutSummaryKpiItem({ + title, + value, + link, + isLoading, +}: { + title: string; + value: string; + link?: { + label: string; + href: string; + }; + isLoading: boolean; +}) { + const { euiTheme } = useEuiTheme(); + + return ( + + + + +
{title}
+ + {link ? ( + + + {link.label} + + + ) : null} + + + + +

{value}

+ + + + + + ); +} + +export function FlyoutSummaryKpiItemLoading({ title }: { title: string }) { + return ( + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpis.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpis.tsx new file mode 100644 index 00000000000000..f377f139eb0991 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary_kpis.tsx @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useMemo } from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { _IGNORED } from '../../../../common/es_fields'; + +import { DataStreamDetails } from '../../../../common/api_types'; +import { useKibanaContextForPlugin } from '../../../utils'; +import { useLinkToLogsExplorer } from '../../../hooks'; +import { FlyoutDataset, TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; +import { FlyoutSummaryKpiItem, FlyoutSummaryKpiItemLoading } from './flyout_summary_kpi_item'; +import { getSummaryKpis } from './get_summary_kpis'; + +export function FlyoutSummaryKpis({ + dataStreamStat, + dataStreamDetails, + isLoading, + timeRange, +}: { + dataStreamStat: FlyoutDataset; + dataStreamDetails?: DataStreamDetails; + isLoading: boolean; + timeRange: TimeRangeConfig; +}) { + const { + services: { observabilityShared }, + } = useKibanaContextForPlugin(); + const hostsLocator = observabilityShared.locators.infra.hostsLocator; + + const logsExplorerLinkProps = useLinkToLogsExplorer({ + dataStreamStat, + query: { language: 'kuery', query: `${_IGNORED}: *` }, + }); + + const kpis = useMemo( + () => + getSummaryKpis({ + dataStreamDetails, + timeRange, + degradedDocsHref: logsExplorerLinkProps.href, + hostsLocator, + }), + [dataStreamDetails, logsExplorerLinkProps, hostsLocator, timeRange] + ); + + return ( + + + {kpis.map((kpi) => ( + + + + ))} + + + ); +} + +export function FlyoutSummaryKpisLoading() { + return ( + + + {getSummaryKpis({}).map(({ title }) => ( + + + + ))} + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.test.ts new file mode 100644 index 00000000000000..fb47ddcddbfa74 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { formatNumber } from '@elastic/eui'; +import type { useKibanaContextForPlugin } from '../../../utils'; +import { TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; + +import { + BYTE_NUMBER_FORMAT, + DEFAULT_DATEPICKER_REFRESH, + DEFAULT_TIME_RANGE, + MAX_HOSTS_METRIC_VALUE, +} from '../../../../common/constants'; +import { + flyoutDegradedDocsText, + flyoutDocsCountTotalText, + flyoutHostsText, + flyoutServicesText, + flyoutShowAllText, + flyoutSizeText, +} from '../../../../common/translations'; +import { getSummaryKpis } from './get_summary_kpis'; + +const dataStreamDetails = { + services: { + service1: ['service1Instance1', 'service1Instance2'], + service2: ['service2Instance1'], + }, + docsCount: 1000, + sizeBytes: 5000, + hosts: { + host1: ['host1Instance1', 'host1Instance2'], + host2: ['host2Instance1'], + }, + degradedDocsCount: 200, +}; + +const timeRange: TimeRangeConfig = { + ...DEFAULT_TIME_RANGE, + refresh: DEFAULT_DATEPICKER_REFRESH, + from: 'now-15m', + to: 'now', +}; + +const degradedDocsHref = 'http://exploratory-view/degraded-docs'; +const hostsRedirectUrl = 'http://hosts/metric/'; + +const hostsLocator = { + getRedirectUrl: () => hostsRedirectUrl, +} as unknown as ReturnType< + typeof useKibanaContextForPlugin +>['services']['observabilityShared']['locators']['infra']['hostsLocator']; + +describe('getSummaryKpis', () => { + it('should return the correct KPIs', () => { + const result = getSummaryKpis({ + dataStreamDetails, + timeRange, + degradedDocsHref, + hostsLocator, + }); + + expect(result).toEqual([ + { + title: flyoutDocsCountTotalText, + value: '1,000', + }, + { + title: flyoutSizeText, + value: formatNumber(dataStreamDetails.sizeBytes ?? 0, BYTE_NUMBER_FORMAT), + }, + { + title: flyoutServicesText, + value: '3', + link: undefined, + }, + { + title: flyoutHostsText, + value: '3', + link: { + label: flyoutShowAllText, + href: hostsRedirectUrl, + }, + }, + { + title: flyoutDegradedDocsText, + value: '200', + link: { + label: flyoutShowAllText, + href: degradedDocsHref, + }, + }, + ]); + }); + + it('show X+ if number of hosts or services exceed MAX_HOSTS_METRIC_VALUE', () => { + const services = { + service1: new Array(MAX_HOSTS_METRIC_VALUE + 1) + .fill('service1Instance') + .map((_, i) => `service1Instance${i}`), + }; + + const host3 = new Array(MAX_HOSTS_METRIC_VALUE + 1) + .fill('host3Instance') + .map((_, i) => `host3Instance${i}`); + + const detailsWithMaxPlusHosts = { + ...dataStreamDetails, + services, + hosts: { ...dataStreamDetails.hosts, host3 }, + }; + + const result = getSummaryKpis({ + dataStreamDetails: detailsWithMaxPlusHosts, + timeRange, + degradedDocsHref, + hostsLocator, + }); + + expect(result).toEqual([ + { + title: flyoutDocsCountTotalText, + value: '1,000', + }, + { + title: flyoutSizeText, + value: formatNumber(dataStreamDetails.sizeBytes ?? 0, BYTE_NUMBER_FORMAT), + }, + { + title: flyoutServicesText, + value: '50+', + link: undefined, + }, + { + title: flyoutHostsText, + value: '54+', + link: { + label: flyoutShowAllText, + href: hostsRedirectUrl, + }, + }, + { + title: flyoutDegradedDocsText, + value: '200', + link: { + label: flyoutShowAllText, + href: degradedDocsHref, + }, + }, + ]); + }); +}); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.ts b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.ts new file mode 100644 index 00000000000000..25d0bf0584bfa3 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/get_summary_kpis.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { formatNumber } from '@elastic/eui'; +import { + BYTE_NUMBER_FORMAT, + DEFAULT_DATEPICKER_REFRESH, + DEFAULT_TIME_RANGE, + MAX_HOSTS_METRIC_VALUE, + NUMBER_FORMAT, +} from '../../../../common/constants'; +import { + flyoutDegradedDocsText, + flyoutDocsCountTotalText, + flyoutHostsText, + flyoutServicesText, + flyoutShowAllText, + flyoutSizeText, +} from '../../../../common/translations'; +import { DataStreamDetails } from '../../../../common/api_types'; +import { useKibanaContextForPlugin } from '../../../utils'; +import { TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; + +export function getSummaryKpis({ + dataStreamDetails, + timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH }, + degradedDocsHref, + hostsLocator, +}: { + dataStreamDetails?: DataStreamDetails; + timeRange?: TimeRangeConfig; + degradedDocsHref?: string; + hostsLocator?: ReturnType< + typeof useKibanaContextForPlugin + >['services']['observabilityShared']['locators']['infra']['hostsLocator']; +}): Array<{ title: string; value: string; link?: { label: string; href: string } }> { + const services = dataStreamDetails?.services ?? {}; + const serviceKeys = Object.keys(services); + const countOfServices = serviceKeys + .map((key: string) => services[key].length) + .reduce((a, b) => a + b, 0); + const servicesLink = undefined; // TODO: Add link to APM services page when possible + + const degradedDocsLink = degradedDocsHref + ? { + label: flyoutShowAllText, + href: degradedDocsHref, + } + : undefined; + + return [ + { + title: flyoutDocsCountTotalText, + value: formatNumber(dataStreamDetails?.docsCount ?? 0, NUMBER_FORMAT), + }, + // dataStreamDetails.sizeBytes = null indicates it's Serverless where `_stats` API isn't available + ...(dataStreamDetails?.sizeBytes !== null // Only show when not in Serverless + ? [ + { + title: flyoutSizeText, + value: formatNumber(dataStreamDetails?.sizeBytes ?? 0, BYTE_NUMBER_FORMAT), + }, + ] + : []), + { + title: flyoutServicesText, + value: formatMetricValueForMax(countOfServices, MAX_HOSTS_METRIC_VALUE, NUMBER_FORMAT), + link: servicesLink, + }, + getHostsKpi(dataStreamDetails?.hosts, timeRange, hostsLocator), + { + title: flyoutDegradedDocsText, + value: formatNumber(dataStreamDetails?.degradedDocsCount ?? 0, NUMBER_FORMAT), + link: degradedDocsLink, + }, + ]; +} + +function getHostsKpi( + dataStreamHosts: DataStreamDetails['hosts'], + timeRange: TimeRangeConfig, + hostsLocator?: ReturnType< + typeof useKibanaContextForPlugin + >['services']['observabilityShared']['locators']['infra']['hostsLocator'] +) { + const hosts = dataStreamHosts ?? {}; + const hostKeys = Object.keys(hosts); + const countOfHosts = hostKeys + .map((key: string) => hosts[key].length) + .reduce( + ({ count, anyHostExceedsMax }, hostCount) => ({ + count: count + hostCount, + anyHostExceedsMax: anyHostExceedsMax || hostCount > MAX_HOSTS_METRIC_VALUE, + }), + { count: 0, anyHostExceedsMax: false } + ); + + // Create a query so from hostKeys so that (key: value OR key: value2) + const hostsKuery = hostKeys + .filter((key) => hosts[key].length > 0) + .map((key) => hosts[key].map((value) => `${key}: "${value}"`).join(' OR ')) + .join(' OR '); + const hostsUrl = hostsLocator?.getRedirectUrl({ + query: { language: 'kuery', query: hostsKuery }, + dateRange: { from: timeRange.from, to: timeRange.to }, + limit: countOfHosts.count, + }); + const hostsLink = hostsUrl + ? { + label: flyoutShowAllText, + href: hostsUrl, + } + : undefined; + + return { + title: flyoutHostsText, + value: formatMetricValueForMax( + countOfHosts.anyHostExceedsMax ? countOfHosts.count + 1 : countOfHosts.count, + countOfHosts.count, + NUMBER_FORMAT + ), + link: hostsLink, + }; +} + +/** + * Formats a metric value to show a '+' sign if it's above a max value e.g. 50+ + */ +function formatMetricValueForMax(value: number, max: number, numberFormat: string): string { + const exceedsMax = value > max; + const valueToShow = exceedsMax ? max : value; + return `${formatNumber(valueToShow, numberFormat)}${exceedsMax ? '+' : ''}`; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_actions_menu.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_actions_menu.tsx index 7a3b2715f17828..9705cd8b5ffd27 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_actions_menu.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_actions_menu.tsx @@ -14,12 +14,18 @@ import { EuiContextMenuPanelDescriptor, EuiContextMenuPanelItemDescriptor, EuiPopover, + EuiSkeletonRectangle, } from '@elastic/eui'; import { css } from '@emotion/react'; import { RouterLinkProps } from '@kbn/router-utils/src/get_router_link_props'; import { Integration } from '../../../common/data_streams_stats/integration'; import { useDatasetQualityFlyout } from '../../hooks'; import { useFlyoutIntegrationActions } from '../../hooks/use_flyout_integration_actions'; + +const integrationActionsText = i18n.translate('xpack.datasetQuality.flyoutIntegrationActionsText', { + defaultMessage: 'Integration actions', +}); + const seeIntegrationText = i18n.translate('xpack.datasetQuality.flyoutSeeIntegrationActionText', { defaultMessage: 'See integration', }); @@ -32,7 +38,13 @@ const viewDashboardsText = i18n.translate('xpack.datasetQuality.flyoutViewDashbo defaultMessage: 'View dashboards', }); -export function IntegrationActionsMenu({ integration }: { integration: Integration }) { +export function IntegrationActionsMenu({ + integration, + dashboardsLoading, +}: { + integration: Integration; + dashboardsLoading: boolean; +}) { const { type, name } = useDatasetQualityFlyout().dataStreamStat!; const { dashboards = [], version, name: integrationName } = integration; const { @@ -46,6 +58,8 @@ export function IntegrationActionsMenu({ integration }: { integration: Integrati const actionButton = ( , + 'data-test-subj': 'datasetQualityFlyoutIntegrationActionDashboardsLoading', + disabled: true, + }); } const panel: EuiContextMenuPanelDescriptor[] = [ @@ -150,6 +171,7 @@ export function IntegrationActionsMenu({ integration }: { integration: Integrati name, type, version, + dashboardsLoading, ]); return ( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_summary.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_summary.tsx index d77c68e4ac33ba..1450e830eac1e7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_summary.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/integration_summary.tsx @@ -18,10 +18,18 @@ import { IntegrationIcon } from '../common'; import { FieldsList } from './fields_list'; import { IntegrationActionsMenu } from './integration_actions_menu'; -export function IntegrationSummary({ integration }: { integration: Integration }) { +export function IntegrationSummary({ + integration, + dashboardsLoading, +}: { + integration: Integration; + dashboardsLoading: boolean; +}) { const { name, version } = integration; - const integrationActionsMenu = ; + const integrationActionsMenu = ( + + ); return ( ), + isLoading: false, }, { fieldTitle: flyoutIntegrationVersionText, fieldValue: version, + isLoading: false, }, ]} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx index 15b1ba311db0ad..aab764f6f6396a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx @@ -6,6 +6,7 @@ */ import { useSelector } from '@xstate/react'; +import { useMemo } from 'react'; import { useDatasetQualityContext } from '../components/dataset_quality/context'; import { useKibanaContextForPlugin } from '../utils'; @@ -18,25 +19,39 @@ export const useDatasetQualityFlyout = () => { const { dataset: dataStreamStat, + datasetSettings: dataStreamSettings, datasetDetails: dataStreamDetails, insightsTimeRange, breakdownField, } = useSelector(service, (state) => state.context.flyout); const { timeRange } = useSelector(service, (state) => state.context.filters); - const dataStreamDetailsLoading = useSelector( - service, - (state) => - state.matches('datasets.loaded.flyoutOpen.fetching') || - state.matches('flyout.initializing.dataStreamDetails.fetching') + const dataStreamDetailsLoading = useSelector(service, (state) => + state.matches('flyout.initializing.dataStreamDetails.fetching') ); + const dataStreamSettingsLoading = useSelector(service, (state) => + state.matches('flyout.initializing.dataStreamSettings.fetching') + ); + + const datasetIntegrationsLoading = useSelector(service, (state) => + state.matches('flyout.initializing.integrationDashboards.fetching') + ); + + const loadingState = useMemo(() => { + return { + dataStreamDetailsLoading, + dataStreamSettingsLoading, + datasetIntegrationsLoading, + }; + }, [dataStreamDetailsLoading, dataStreamSettingsLoading, datasetIntegrationsLoading]); return { dataStreamStat, + dataStreamSettings, dataStreamDetails, - dataStreamDetailsLoading, fieldFormats, timeRange: insightsTimeRange ?? timeRange, breakdownField, + loadingState, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 116e15bd92c933..2d1980c74daeee 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -61,10 +61,6 @@ export const useDatasetQualityTable = () => { const datasets = useSelector(service, (state) => state.context.datasets); - const isDatasetQualityPageIdle = useSelector(service, (state) => - state.matches('datasets.loaded.idle') - ); - const toggleInactiveDatasets = useCallback( () => service.send({ type: 'TOGGLE_INACTIVE_DATASETS' }), [service] @@ -86,7 +82,7 @@ export const useDatasetQualityTable = () => { return; } - if (isDatasetQualityPageIdle) { + if (!flyout?.insightsTimeRange) { service.send({ type: 'OPEN_FLYOUT', dataset: selectedDataset, @@ -99,7 +95,7 @@ export const useDatasetQualityTable = () => { dataset: selectedDataset, }); }, - [flyout?.dataset?.rawName, isDatasetQualityPageIdle, service] + [flyout?.dataset?.rawName, flyout?.insightsTimeRange, service] ); const isActive = useCallback( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.tsx index 2e37f84a0abb19..6a92249477cb2b 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.tsx @@ -4,12 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import { useCallback, useState, useMemo, useEffect } from 'react'; import { Action } from '@kbn/ui-actions-plugin/public'; +import { fieldSupportsBreakdown } from '@kbn/unified-histogram-plugin/public'; import { i18n } from '@kbn/i18n'; import { useEuiTheme } from '@elastic/eui'; -import { DataViewField } from '@kbn/data-views-plugin/common'; +import { type DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import { useDatasetQualityContext } from '../components/dataset_quality/context'; import { DEFAULT_LOGS_DATA_VIEW } from '../../common/constants'; import { indexNameToDataStreamParts } from '../../common/utils'; import { getLensAttributes } from '../components/flyout/degraded_docs_trend/lens_attributes'; @@ -34,37 +37,47 @@ const ACTION_OPEN_IN_LENS = 'ACTION_OPEN_IN_LENS'; interface DegradedDocsChartDeps { dataStream?: string; - breakdownDataViewField?: DataViewField; + breakdownField?: string; } -export const useDegradedDocsChart = ({ - dataStream, - breakdownDataViewField, -}: DegradedDocsChartDeps) => { +export const useDegradedDocsChart = ({ dataStream }: DegradedDocsChartDeps) => { + const { euiTheme } = useEuiTheme(); const { services: { lens }, } = useKibanaContextForPlugin(); - const { euiTheme } = useEuiTheme(); + const { service } = useDatasetQualityContext(); - const { dataStreamStat, timeRange } = useDatasetQualityFlyout(); + const { dataStreamStat, timeRange, breakdownField } = useDatasetQualityFlyout(); const [isChartLoading, setIsChartLoading] = useState(undefined); const [attributes, setAttributes] = useState | undefined>( undefined ); - const datasetTypeIndexPattern = dataStream - ? `${indexNameToDataStreamParts(dataStream).type}-*-*` - : undefined; const { dataView } = useCreateDataView({ - indexPatternString: datasetTypeIndexPattern ?? DEFAULT_LOGS_DATA_VIEW, + indexPatternString: getDataViewIndexPattern(dataStream), }); + + const breakdownDataViewField = useMemo( + () => getDataViewField(dataView, breakdownField), + [breakdownField, dataView] + ); const filterQuery = `_index: ${dataStream ?? 'match-none'}`; const handleChartLoading = (isLoading: boolean) => { setIsChartLoading(isLoading); }; + const handleBreakdownFieldChange = useCallback( + (field: DataViewField | undefined) => { + service.send({ + type: 'BREAKDOWN_FIELD_CHANGE', + breakdownField: field?.name ?? null, + }); + }, + [service] + ); + useEffect(() => { if (dataView) { const lensAttributes = getLensAttributes({ @@ -111,6 +124,7 @@ export const useDegradedDocsChart = ({ dataStreamStat: dataStreamStat!, query: { language: 'kuery', query: '_ignored:*' }, timeRangeConfig: timeRange, + breakdownField: breakdownDataViewField?.name, }); const getOpenInLogsExplorerAction = useMemo(() => { @@ -141,11 +155,27 @@ export const useDegradedDocsChart = ({ return { attributes, dataView, - filterQuery, + breakdown: { + dataViewField: breakdownDataViewField, + fieldSupportsBreakdown: breakdownDataViewField + ? fieldSupportsBreakdown(breakdownDataViewField) + : true, + onChange: handleBreakdownFieldChange, + }, extraActions, isChartLoading, - handleChartLoading, + onChartLoading: handleChartLoading, setAttributes, setIsChartLoading, }; }; + +function getDataViewIndexPattern(dataStream: string | undefined) { + return dataStream ? `${indexNameToDataStreamParts(dataStream).type}-*-*` : DEFAULT_LOGS_DATA_VIEW; +} + +function getDataViewField(dataView: DataView | undefined, fieldName: string | undefined) { + return fieldName && dataView + ? dataView.fields.find((field) => field.name === fieldName) + : undefined; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_link_to_logs_explorer.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_link_to_logs_explorer.ts index c1a857da1cc71f..ff442f970ab21a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_link_to_logs_explorer.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_link_to_logs_explorer.ts @@ -21,10 +21,12 @@ export const useLinkToLogsExplorer = ({ dataStreamStat, query, timeRangeConfig, + breakdownField, }: { dataStreamStat: DataStreamStat | FlyoutDataset; query?: Query | AggregateQuery; timeRangeConfig?: TimeRangeConfig; + breakdownField?: string; }) => { const { services: { share }, @@ -48,6 +50,7 @@ export const useLinkToLogsExplorer = ({ values: [dataStreamStat.namespace], }, }, + breakdownField, }; const singleDatasetLocator = diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 1dd5b2d2220cdd..3b95814397f3ad 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,26 +8,50 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { + getDataStreamsSettingsResponseRt, getDataStreamsDetailsResponseRt, integrationDashboardsRT, } from '../../../common/api_types'; import { GetDataStreamsStatsError, + GetDataStreamSettingsParams, + GetDataStreamSettingsResponse, GetDataStreamDetailsParams, GetDataStreamDetailsResponse, GetIntegrationDashboardsParams, GetIntegrationDashboardsResponse, } from '../../../common/data_streams_stats'; -import { DataStreamDetails } from '../../../common/data_streams_stats'; +import { DataStreamDetails, DataStreamSettings } from '../../../common/data_streams_stats'; import { IDataStreamDetailsClient } from './types'; export class DataStreamDetailsClient implements IDataStreamDetailsClient { constructor(private readonly http: HttpStart) {} - public async getDataStreamDetails({ dataStream }: GetDataStreamDetailsParams) { + public async getDataStreamSettings({ dataStream }: GetDataStreamSettingsParams) { + const response = await this.http + .get( + `/internal/dataset_quality/data_streams/${dataStream}/settings` + ) + .catch((error) => { + throw new GetDataStreamsStatsError(`Failed to fetch data stream settings": ${error}`); + }); + + const dataStreamSettings = decodeOrThrow( + getDataStreamsSettingsResponseRt, + (message: string) => + new GetDataStreamsStatsError(`Failed to decode data stream settings response: ${message}"`) + )(response); + + return dataStreamSettings as DataStreamSettings; + } + + public async getDataStreamDetails({ dataStream, start, end }: GetDataStreamDetailsParams) { const response = await this.http .get( - `/internal/dataset_quality/data_streams/${dataStream}/details` + `/internal/dataset_quality/data_streams/${dataStream}/details`, + { + query: { start, end }, + } ) .catch((error) => { throw new GetDataStreamsStatsError(`Failed to fetch data stream details": ${error}`); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index 068b36bab4fb1c..f85e05d0179f5e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -7,6 +7,8 @@ import { HttpStart } from '@kbn/core/public'; import { + GetDataStreamSettingsParams, + DataStreamSettings, GetDataStreamDetailsParams, DataStreamDetails, GetIntegrationDashboardsParams, @@ -24,6 +26,7 @@ export interface DataStreamDetailsServiceStartDeps { } export interface IDataStreamDetailsClient { + getDataStreamSettings(params: GetDataStreamSettingsParams): Promise; getDataStreamDetails(params: GetDataStreamDetailsParams): Promise; getIntegrationDashboards( params: GetIntegrationDashboardsParams diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index 4d399552ec5e8f..3d88e6645fe479 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -26,6 +26,15 @@ export const fetchDatasetDetailsFailedNotifier = (toasts: IToasts, error: Error) }); }; +export const fetchDatasetSettingsFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchDatasetSettingsFailed', { + defaultMessage: "Dataset settings couldn't be loaded.", + }), + text: error.message, + }); +}; + export const fetchDegradedStatsFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ title: i18n.translate('xpack.datasetQuality.fetchDegradedStatsFailed', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 39d630a7aa7707..f13e2c7340851c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -12,6 +12,7 @@ import { Integration } from '../../../../common/data_streams_stats/integration'; import { IDataStreamDetailsClient } from '../../../services/data_stream_details'; import { DashboardType, + DataStreamSettings, DataStreamDetails, DataStreamStat, GetDataStreamsStatsQuery, @@ -24,6 +25,7 @@ import { IDataStreamsStatsClient } from '../../../services/data_streams_stats'; import { generateDatasets } from '../../../utils'; import { DEFAULT_CONTEXT } from './defaults'; import { + fetchDatasetSettingsFailedNotifier, fetchDatasetDetailsFailedNotifier, fetchDatasetStatsFailedNotifier, fetchDegradedStatsFailedNotifier, @@ -173,6 +175,27 @@ export const createPureDatasetQualityControllerStateMachine = ( initializing: { type: 'parallel', states: { + dataStreamSettings: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDataStreamSettings', + onDone: { + target: 'done', + actions: ['storeDataStreamSettings'], + }, + onError: { + target: 'done', + actions: ['notifyFetchDatasetSettingsFailed'], + }, + }, + }, + done: { + type: 'final', + }, + }, + }, dataStreamDetails: { initial: 'fetching', states: { @@ -185,12 +208,20 @@ export const createPureDatasetQualityControllerStateMachine = ( }, onError: { target: 'done', - actions: ['fetchDatasetDetailsFailedNotifier'], + actions: ['notifyFetchDatasetDetailsFailed'], }, }, }, done: { - type: 'final', + on: { + UPDATE_INSIGHTS_TIME_RANGE: { + target: 'fetching', + actions: ['storeFlyoutOptions'], + }, + BREAKDOWN_FIELD_CHANGE: { + actions: ['storeFlyoutOptions'], + }, + }, }, }, }, @@ -226,12 +257,6 @@ export const createPureDatasetQualityControllerStateMachine = ( target: 'closed', actions: ['resetFlyoutOptions'], }, - UPDATE_INSIGHTS_TIME_RANGE: { - actions: ['storeFlyoutOptions'], - }, - BREAKDOWN_FIELD_CHANGE: { - actions: ['storeFlyoutOptions'], - }, }, }, closed: { @@ -328,28 +353,25 @@ export const createPureDatasetQualityControllerStateMachine = ( : {}; }), storeFlyoutOptions: assign((context, event) => { - return 'dataset' in event - ? { - flyout: { - ...context.flyout, - dataset: event.dataset as FlyoutDataset, - }, - } - : 'timeRange' in event - ? { - flyout: { - ...context.flyout, - insightsTimeRange: event.timeRange, - }, - } - : 'breakdownField' in event - ? { - flyout: { - ...context.flyout, - breakdownField: event.breakdownField ?? undefined, - }, - } - : {}; + const insightsTimeRange = + 'timeRange' in event + ? event.timeRange + : context.flyout?.insightsTimeRange ?? context.filters?.timeRange; + const dataset = + 'dataset' in event ? (event.dataset as FlyoutDataset) : context.flyout?.dataset; + const breakdownField = + 'breakdownField' in event + ? event.breakdownField ?? undefined + : context.flyout?.breakdownField; + + return { + flyout: { + ...context.flyout, + dataset, + insightsTimeRange, + breakdownField, + }, + }; }), resetFlyoutOptions: assign((_context, _event) => ({ flyout: undefined })), storeDataStreamStats: assign((_context, event) => { @@ -366,6 +388,16 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), + storeDataStreamSettings: assign((context, event) => { + return 'data' in event + ? { + flyout: { + ...context.flyout, + datasetSettings: (event.data ?? {}) as DataStreamSettings, + }, + } + : {}; + }), storeDatasetDetails: assign((context, event) => { return 'data' in event ? { @@ -438,6 +470,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchDatasetStatsFailedNotifier(toasts, event.data), notifyFetchDegradedStatsFailed: (_context, event: DoneInvokeEvent) => fetchDegradedStatsFailedNotifier(toasts, event.data), + notifyFetchDatasetSettingsFailed: (_context, event: DoneInvokeEvent) => + fetchDatasetSettingsFailedNotifier(toasts, event.data), notifyFetchDatasetDetailsFailed: (_context, event: DoneInvokeEvent) => fetchDatasetDetailsFailedNotifier(toasts, event.data), notifyFetchIntegrationDashboardsFailed: (_context, event: DoneInvokeEvent) => @@ -466,14 +500,34 @@ export const createDatasetQualityControllerStateMachine = ({ type: context.type as GetIntegrationsParams['query']['type'], }); }, - loadDataStreamDetails: (context) => { + loadDataStreamSettings: (context) => { if (!context.flyout.dataset) { + fetchDatasetSettingsFailedNotifier(toasts, new Error(noDatasetSelected)); + + return Promise.resolve({}); + } + + const { type, name: dataset, namespace } = context.flyout.dataset; + + return dataStreamDetailsClient.getDataStreamSettings({ + dataStream: dataStreamPartsToIndexName({ + type: type as DataStreamType, + dataset, + namespace, + }), + }); + }, + loadDataStreamDetails: (context) => { + if (!context.flyout.dataset || !context.flyout.insightsTimeRange) { fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected)); return Promise.resolve({}); } const { type, name: dataset, namespace } = context.flyout.dataset; + const { startDate: start, endDate: end } = getDateISORange( + context.flyout.insightsTimeRange + ); return dataStreamDetailsClient.getDataStreamDetails({ dataStream: dataStreamPartsToIndexName({ @@ -481,6 +535,8 @@ export const createDatasetQualityControllerStateMachine = ({ dataset, namespace, }), + start, + end, }); }, loadIntegrationDashboards: (context) => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 5af18adf8313d9..3cfb3e9c5a5486 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -13,11 +13,12 @@ import { DegradedDocsStat } from '../../../../common/data_streams_stats/malforme import { DashboardType, DataStreamDegradedDocsStatServiceResponse, + DataStreamSettings, DataStreamDetails, DataStreamStatServiceResponse, IntegrationsResponse, + DataStreamStat, } from '../../../../common/data_streams_stats'; -import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat'; export type FlyoutDataset = Omit< DataStreamStat, @@ -53,6 +54,7 @@ export interface WithTableOptions { export interface WithFlyoutOptions { flyout: { dataset?: FlyoutDataset; + datasetSettings?: DataStreamSettings; datasetDetails?: DataStreamDetails; insightsTimeRange?: TimeRangeConfig; breakdownField?: string; @@ -103,14 +105,6 @@ export type DatasetQualityControllerTypeState = value: 'datasets.loaded.idle'; context: DefaultDatasetQualityStateContext; } - | { - value: 'datasets.loaded.flyoutOpen.fetching'; - context: DefaultDatasetQualityStateContext; - } - | { - value: 'datasets.loaded.flyoutOpen'; - context: DefaultDatasetQualityStateContext; - } | { value: 'degradedDocs.fetching'; context: DefaultDatasetQualityStateContext; @@ -123,6 +117,10 @@ export type DatasetQualityControllerTypeState = value: 'integrations.fetching'; context: DefaultDatasetQualityStateContext; } + | { + value: 'flyout.initializing.dataStreamSettings.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'flyout.initializing.dataStreamDetails.fetching'; context: DefaultDatasetQualityStateContext; @@ -185,6 +183,8 @@ export type DatasetQualityControllerEvent = } | DoneInvokeEvent | DoneInvokeEvent + | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/types.ts index b9755a8c286281..7da7875978740e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/types.ts @@ -12,8 +12,10 @@ import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { LensPublicStart } from '@kbn/lens-plugin/public'; -import type { DatasetQualityProps } from './components/dataset_quality'; +import type { ObservabilitySharedPluginSetup } from '@kbn/observability-shared-plugin/public'; + import type { CreateDatasetQualityController } from './controller'; +import type { DatasetQualityProps } from './components/dataset_quality'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface DatasetQualityPluginSetup {} @@ -30,6 +32,7 @@ export interface DatasetQualityStartDeps { unifiedSearch: UnifiedSearchPublicPluginStart; lens: LensPublicStart; dataViews: DataViewsPublicPluginStart; + observabilityShared: ObservabilitySharedPluginSetup; } export interface DatasetQualitySetupDeps { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts index faaf3d1c44b6f8..1b850ff2c9fd96 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_details.test.ts @@ -5,203 +5,175 @@ * 2.0. */ +import { SearchTotalHitsRelation } from '@elastic/elasticsearch/lib/api/types'; import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; +import { + findInventoryFields, + InventoryItemType, + inventoryModels, +} from '@kbn/metrics-data-access-plugin/common'; import { getDataStreamDetails } from '.'; const accessLogsDataStream = 'logs-nginx.access-default'; const errorLogsDataStream = 'logs-nginx.error-default'; -const dateStr1 = '1702998651925'; // .ds-logs-nginx.access-default-2023.12.19-000001 -const dateStr2 = '1703110671019'; // .ds-logs-nginx.access-default-2023.12.20-000002 -const dateStr3 = '1702998866744'; // .ds-logs-nginx.error-default-2023.12.19-000001 + +const defaultSummaryStats = { + degradedDocsCount: 98841, + docsCount: 617680, + hosts: { + 'aws.rds.db_instance.arn': [], + 'aws.s3.bucket.name': [], + 'aws.sqs.queue.name': [], + 'cloud.instance.id': ['0000000000009121', '0000000000009127', '0000000000009133'], + 'container.id': [], + 'host.name': ['synth-host'], + 'kubernetes.pod.uid': [], + }, + services: { + 'service.name': ['synth-service-0', 'synth-service-1', 'synth-service-2'], + }, + sizeBytes: 72596354, +}; + +const start = Number(new Date('2020-01-01T00:00:00.000Z')); +const end = Number(new Date('2020-01-30T00:00:00.000Z')); describe('getDataStreamDetails', () => { afterAll(() => { jest.clearAllMocks(); }); - it('throws error if index is not found', async () => { + it('returns {} if index is not found', async () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); esClientMock.indices.getSettings.mockRejectedValue(MOCK_INDEX_ERROR); + esClientMock.search.mockRejectedValue(MOCK_INDEX_ERROR); + + const dataStreamDetails = await getDataStreamDetails({ + esClient: esClientMock, + dataStream: 'non-existent', + start, + end, + }); - try { - await getDataStreamDetails({ - esClient: esClientMock, - dataStream: 'non-existent', - }); - } catch (e) { - expect(e).toBe(MOCK_INDEX_ERROR); - } + expect(dataStreamDetails).toEqual({}); }); - it('returns creation date of a data stream', async () => { + it('returns summary of a data stream', async () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); - esClientMock.indices.getSettings.mockReturnValue( - Promise.resolve(MOCK_NGINX_ERROR_INDEX_SETTINGS) - ); + esClientMock.indices.stats.mockReturnValue(Promise.resolve(MOCK_STATS_RESPONSE)); + esClientMock.search.mockReturnValue(Promise.resolve(MOCK_SEARCH_RESPONSE)); const dataStreamDetails = await getDataStreamDetails({ esClient: esClientMock, dataStream: errorLogsDataStream, + start, + end, }); - expect(dataStreamDetails).toEqual({ createdOn: Number(dateStr3) }); + + expect(dataStreamDetails).toEqual(defaultSummaryStats); }); - it('returns the earliest creation date of a data stream with multiple backing indices', async () => { + it('returns the correct service.name list', async () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); - esClientMock.indices.getSettings.mockReturnValue( - Promise.resolve(MOCK_NGINX_ACCESS_INDEX_SETTINGS) + esClientMock.indices.stats.mockReturnValue(Promise.resolve(MOCK_STATS_RESPONSE)); + + const serviceName = 'service.name'; + const testServiceName = ['tst-srv-0', 'tst-srv-1']; + const mockSearchResponse = { ...MOCK_SEARCH_RESPONSE }; + mockSearchResponse.aggregations[serviceName].buckets = testServiceName.map((name) => ({ + key: name, + doc_count: 1, + })); + esClientMock.search.mockReturnValue(Promise.resolve(MOCK_SEARCH_RESPONSE)); + + const dataStreamDetails = await getDataStreamDetails({ + esClient: esClientMock, + dataStream: accessLogsDataStream, + start, + end, + }); + expect(dataStreamDetails.services).toEqual({ [serviceName]: testServiceName }); + }); + + it('returns the correct host.name list', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.stats.mockReturnValue(Promise.resolve(MOCK_STATS_RESPONSE)); + + const hostName = 'host.name'; + const testHostName = ['tst-host-0', 'tst-host-1']; + const hostFields = inventoryModels.map( + (model) => findInventoryFields(model.id as InventoryItemType).id ); + const mockSearchResponse = { ...MOCK_SEARCH_RESPONSE }; + // Make all hosts buckets to [] + hostFields.forEach((field) => { + mockSearchResponse.aggregations[field as 'host.name'] = { buckets: [] } as any; + }); + + // Set the host.name buckets to testHostName + mockSearchResponse.aggregations[hostName].buckets = testHostName.map((name) => ({ + key: name, + doc_count: 1, + })); + + esClientMock.search.mockReturnValue(Promise.resolve(MOCK_SEARCH_RESPONSE)); + const dataStreamDetails = await getDataStreamDetails({ esClient: esClientMock, dataStream: accessLogsDataStream, + start, + end, }); - expect(dataStreamDetails).toEqual({ createdOn: Number(dateStr1) }); + + // Expect all host fields to be empty + const emptyHosts = hostFields.reduce((acc, field) => ({ ...acc, [field]: [] }), {}); + + expect(dataStreamDetails.hosts).toEqual({ ...emptyHosts, [hostName]: testHostName }); }); -}); -const MOCK_NGINX_ACCESS_INDEX_SETTINGS = { - [`.ds-${accessLogsDataStream}-2023.12.19-000001`]: { - settings: { - index: { - mapping: { - total_fields: { - limit: 10000, - }, - ignore_malformed: true, - }, - hidden: true, - provided_name: '.ds-logs-nginx.access-default-2023.12.19-000001', - final_pipeline: '.fleet_final_pipeline-1', - query: { - default_field: [ - 'cloud.account.id', - 'cloud.availability_zone', - 'cloud.instance.id', - 'cloud.instance.name', - 'cloud.machine.type', - 'cloud.provider', - 'cloud.region', - ], - }, - creation_date: dateStr1, - number_of_replicas: '1', - uuid: 'uml9fMQqQUibZi2pKkc5sQ', - version: { - created: '8500007', - }, - lifecycle: { - name: 'logs', - indexing_complete: true, - }, - codec: 'best_compression', - routing: { - allocation: { - include: { - _tier_preference: 'data_hot', - }, - }, - }, - number_of_shards: '1', - default_pipeline: 'logs-nginx.access-1.17.0', - }, - }, - }, - [`.ds-${accessLogsDataStream}-2023.12.20-000002`]: { - settings: { - index: { - mapping: { - total_fields: { - limit: 10000, - }, - ignore_malformed: true, - }, - hidden: true, - provided_name: '.ds-logs-nginx.access-default-2023.12.20-000002', - final_pipeline: '.fleet_final_pipeline-1', - query: { - default_field: [ - 'user.name', - 'user_agent.device.name', - 'user_agent.name', - 'user_agent.original', - 'user_agent.os.full', - 'user_agent.os.name', - 'user_agent.os.version', - 'user_agent.version', - 'nginx.access.remote_ip_list', - ], - }, - creation_date: dateStr2, - number_of_replicas: '1', - uuid: 'il9vJlOXRdiv44wU6WNtUQ', - version: { - created: '8500007', - }, - lifecycle: { - name: 'logs', - }, - codec: 'best_compression', - routing: { - allocation: { - include: { - _tier_preference: 'data_hot', - }, - }, - }, - number_of_shards: '1', - default_pipeline: 'logs-nginx.access-1.17.0', - }, - }, - }, -}; + it('returns correct size in bytes', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); -const MOCK_NGINX_ERROR_INDEX_SETTINGS = { - [`.ds-${errorLogsDataStream}-2023.12.19-000001`]: { - settings: { - index: { - mapping: { - total_fields: { - limit: 10000, - }, - ignore_malformed: true, - }, - hidden: true, - provided_name: '.ds-logs-nginx.error-default-2023.12.19-000001', - final_pipeline: '.fleet_final_pipeline-1', - query: { - default_field: [ - 'host.type', - 'input.type', - 'log.file.path', - 'log.level', - 'ecs.version', - 'message', - 'tags', - ], - }, - creation_date: dateStr3, - number_of_replicas: '1', - uuid: 'fGPYUppSRU62MZ3toF0MkQ', - version: { - created: '8500007', - }, - lifecycle: { - name: 'logs', - }, - codec: 'best_compression', - routing: { - allocation: { - include: { - _tier_preference: 'data_hot', - }, - }, - }, - number_of_shards: '1', - default_pipeline: 'logs-nginx.error-1.17.0', - }, - }, - }, -}; + const docsCount = 536; + const storeDocsCount = 1220; + const storeSizeInBytes = 2048; + const expectedSizeInBytes = Math.ceil((storeSizeInBytes / storeDocsCount) * docsCount); + + const testStatsResponse = { ...MOCK_STATS_RESPONSE }; + testStatsResponse._all.total.docs.count = storeDocsCount; + testStatsResponse._all.total.store.size_in_bytes = storeSizeInBytes; + esClientMock.indices.stats.mockReturnValue(Promise.resolve(testStatsResponse)); + + const mockSearchResponse = { ...MOCK_SEARCH_RESPONSE }; + mockSearchResponse.aggregations.total_count.value = docsCount; + esClientMock.search.mockReturnValue(Promise.resolve(mockSearchResponse)); + + const dataStreamDetails = await getDataStreamDetails({ + esClient: esClientMock, + dataStream: accessLogsDataStream, + start, + end, + }); + expect(dataStreamDetails.sizeBytes).toEqual(expectedSizeInBytes); + }); + + // This covers https://github.com/elastic/kibana/issues/178954 + it('returns size as NaN for when sizeStatsAvailable is false (serverless mode)', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + + esClientMock.indices.stats.mockReturnValue(Promise.resolve(MOCK_STATS_RESPONSE)); + esClientMock.search.mockReturnValue(Promise.resolve(MOCK_SEARCH_RESPONSE)); + + const dataStreamDetails = await getDataStreamDetails({ + esClient: esClientMock, + dataStream: accessLogsDataStream, + start, + end, + sizeStatsAvailable: false, + }); + expect(dataStreamDetails.sizeBytes).toBeNaN(); + }); +}); const MOCK_INDEX_ERROR = { error: { @@ -222,5 +194,142 @@ const MOCK_INDEX_ERROR = { index_uuid: '_na_', index: 'logs-nginx.error-default-01', }, - status: 404, + statusCode: 404, +}; + +const MOCK_SEARCH_RESPONSE = { + took: 2, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 10000, + relation: 'gte' as SearchTotalHitsRelation, + }, + max_score: null, + hits: [], + }, + aggregations: { + total_count: { + value: 617680, + }, + degraded_count: { + doc_count: 98841, + }, + 'service.name': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'synth-service-0', + doc_count: 206116, + }, + { + key: 'synth-service-1', + doc_count: 206012, + }, + { + key: 'synth-service-2', + doc_count: 205552, + }, + ], + }, + 'host.name': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'synth-host', + doc_count: 617680, + }, + ], + }, + 'kubernetes.pod.uid': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + 'container.id': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + 'cloud.instance.id': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 614630, + buckets: [ + { + key: '0000000000009121', + doc_count: 61, + }, + { + key: '0000000000009127', + doc_count: 61, + }, + { + key: '0000000000009133', + doc_count: 61, + }, + ], + }, + 'aws.s3.bucket.name': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + 'aws.rds.db_instance.arn': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + 'aws.sqs.queue.name': { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + }, +}; + +const MOCK_STATS_RESPONSE = { + _shards: { + total: 2, + successful: 2, + failed: 0, + }, + _all: { + primaries: {}, + total: { + docs: { + count: 1235360, + deleted: 0, + }, + shard_stats: { + total_count: 2, + }, + store: { + size_in_bytes: 145192707, + total_data_set_size_in_bytes: 145192707, + reserved_in_bytes: 0, + }, + indexing: { + index_total: 1235059, + index_time_in_millis: 98509, + index_current: 0, + index_failed: 0, + delete_total: 0, + delete_time_in_millis: 0, + delete_current: 0, + noop_update_total: 0, + is_throttled: false, + throttle_time_in_millis: 0, + write_load: 0.00022633763414114222, + }, + }, + }, + indices: {}, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_settings.test.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_settings.test.ts new file mode 100644 index 00000000000000..dd2548d033c714 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/get_data_stream_settings.test.ts @@ -0,0 +1,227 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; + +import { getDataStreamSettings } from '.'; +const accessLogsDataStream = 'logs-nginx.access-default'; +const errorLogsDataStream = 'logs-nginx.error-default'; +const dateStr1 = '1702998651925'; // .ds-logs-nginx.access-default-2023.12.19-000001 +const dateStr2 = '1703110671019'; // .ds-logs-nginx.access-default-2023.12.20-000002 +const dateStr3 = '1702998866744'; // .ds-logs-nginx.error-default-2023.12.19-000001 + +describe('getDataStreamSettings', () => { + afterAll(() => { + jest.clearAllMocks(); + }); + + it('throws error if index is not found', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.getSettings.mockRejectedValue(MOCK_INDEX_ERROR); + + try { + await getDataStreamSettings({ + esClient: esClientMock, + dataStream: 'non-existent', + }); + } catch (e) { + expect(e).toBe(MOCK_INDEX_ERROR); + } + }); + + it('returns creation date of a data stream', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.getSettings.mockReturnValue( + Promise.resolve(MOCK_NGINX_ERROR_INDEX_SETTINGS) + ); + + const dataStreamSettings = await getDataStreamSettings({ + esClient: esClientMock, + dataStream: errorLogsDataStream, + }); + expect(dataStreamSettings).toEqual({ createdOn: Number(dateStr3) }); + }); + + it('returns the earliest creation date of a data stream with multiple backing indices', async () => { + const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + esClientMock.indices.getSettings.mockReturnValue( + Promise.resolve(MOCK_NGINX_ACCESS_INDEX_SETTINGS) + ); + + const dataStreamSettings = await getDataStreamSettings({ + esClient: esClientMock, + dataStream: accessLogsDataStream, + }); + expect(dataStreamSettings).toEqual({ createdOn: Number(dateStr1) }); + }); +}); + +const MOCK_NGINX_ACCESS_INDEX_SETTINGS = { + [`.ds-${accessLogsDataStream}-2023.12.19-000001`]: { + settings: { + index: { + mapping: { + total_fields: { + limit: 10000, + }, + ignore_malformed: true, + }, + hidden: true, + provided_name: '.ds-logs-nginx.access-default-2023.12.19-000001', + final_pipeline: '.fleet_final_pipeline-1', + query: { + default_field: [ + 'cloud.account.id', + 'cloud.availability_zone', + 'cloud.instance.id', + 'cloud.instance.name', + 'cloud.machine.type', + 'cloud.provider', + 'cloud.region', + ], + }, + creation_date: dateStr1, + number_of_replicas: '1', + uuid: 'uml9fMQqQUibZi2pKkc5sQ', + version: { + created: '8500007', + }, + lifecycle: { + name: 'logs', + indexing_complete: true, + }, + codec: 'best_compression', + routing: { + allocation: { + include: { + _tier_preference: 'data_hot', + }, + }, + }, + number_of_shards: '1', + default_pipeline: 'logs-nginx.access-1.17.0', + }, + }, + }, + [`.ds-${accessLogsDataStream}-2023.12.20-000002`]: { + settings: { + index: { + mapping: { + total_fields: { + limit: 10000, + }, + ignore_malformed: true, + }, + hidden: true, + provided_name: '.ds-logs-nginx.access-default-2023.12.20-000002', + final_pipeline: '.fleet_final_pipeline-1', + query: { + default_field: [ + 'user.name', + 'user_agent.device.name', + 'user_agent.name', + 'user_agent.original', + 'user_agent.os.full', + 'user_agent.os.name', + 'user_agent.os.version', + 'user_agent.version', + 'nginx.access.remote_ip_list', + ], + }, + creation_date: dateStr2, + number_of_replicas: '1', + uuid: 'il9vJlOXRdiv44wU6WNtUQ', + version: { + created: '8500007', + }, + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + routing: { + allocation: { + include: { + _tier_preference: 'data_hot', + }, + }, + }, + number_of_shards: '1', + default_pipeline: 'logs-nginx.access-1.17.0', + }, + }, + }, +}; + +const MOCK_NGINX_ERROR_INDEX_SETTINGS = { + [`.ds-${errorLogsDataStream}-2023.12.19-000001`]: { + settings: { + index: { + mapping: { + total_fields: { + limit: 10000, + }, + ignore_malformed: true, + }, + hidden: true, + provided_name: '.ds-logs-nginx.error-default-2023.12.19-000001', + final_pipeline: '.fleet_final_pipeline-1', + query: { + default_field: [ + 'host.type', + 'input.type', + 'log.file.path', + 'log.level', + 'ecs.version', + 'message', + 'tags', + ], + }, + creation_date: dateStr3, + number_of_replicas: '1', + uuid: 'fGPYUppSRU62MZ3toF0MkQ', + version: { + created: '8500007', + }, + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + routing: { + allocation: { + include: { + _tier_preference: 'data_hot', + }, + }, + }, + number_of_shards: '1', + default_pipeline: 'logs-nginx.error-1.17.0', + }, + }, + }, +}; + +const MOCK_INDEX_ERROR = { + error: { + root_cause: [ + { + type: 'index_not_found_exception', + reason: 'no such index [logs-nginx.error-default-01]', + 'resource.type': 'index_or_alias', + 'resource.id': 'logs-nginx.error-default-01', + index_uuid: '_na_', + index: 'logs-nginx.error-default-01', + }, + ], + type: 'index_not_found_exception', + reason: 'no such index [logs-nginx.error-default-01]', + 'resource.type': 'index_or_alias', + 'resource.id': 'logs-nginx.error-default-01', + index_uuid: '_na_', + index: 'logs-nginx.error-default-01', + }, + status: 404, +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 65fa54babcc306..c6ee429c27f960 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -7,28 +7,163 @@ import { badRequest } from '@hapi/boom'; import type { ElasticsearchClient } from '@kbn/core/server'; -import { DataStreamDetails } from '../../../../common/api_types'; +import { + findInventoryFields, + InventoryItemType, + inventoryModels, +} from '@kbn/metrics-data-access-plugin/common'; +import { rangeQuery } from '@kbn/observability-plugin/server'; + +import { MAX_HOSTS_METRIC_VALUE } from '../../../../common/constants'; +import { _IGNORED } from '../../../../common/es_fields'; +import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_types'; +import { createDatasetQualityESClient } from '../../../utils'; import { dataStreamService } from '../../../services'; -export async function getDataStreamDetails(args: { +export async function getDataStreamSettings({ + esClient, + dataStream, +}: { + esClient: ElasticsearchClient; + dataStream: string; +}): Promise { + throwIfInvalidDataStreamParams(dataStream); + + const createdOn = await getDataStreamCreatedOn(esClient, dataStream); + + return { + createdOn, + }; +} + +export async function getDataStreamDetails({ + esClient, + dataStream, + start, + end, + sizeStatsAvailable = true, +}: { esClient: ElasticsearchClient; dataStream: string; + start: number; + end: number; + sizeStatsAvailable?: boolean; // Only Needed to determine whether `_stats` endpoint is available https://github.com/elastic/kibana/issues/178954 }): Promise { - const { esClient, dataStream } = args; + throwIfInvalidDataStreamParams(dataStream); - if (!dataStream?.trim()) { - throw badRequest(`Data Stream name cannot be empty. Received value "${dataStream}"`); + try { + const dataStreamSummaryStats = await getDataStreamSummaryStats( + esClient, + dataStream, + start, + end + ); + + const whenSizeStatsNotAvailable = NaN; // This will indicate size cannot be calculated + const avgDocSizeInBytes = sizeStatsAvailable + ? dataStreamSummaryStats.docsCount > 0 + ? await getAvgDocSizeInBytes(esClient, dataStream) + : 0 + : whenSizeStatsNotAvailable; + const sizeBytes = Math.ceil(avgDocSizeInBytes * dataStreamSummaryStats.docsCount); + + return { + ...dataStreamSummaryStats, + sizeBytes, + }; + } catch (e) { + // Respond with empty object if data stream does not exist + if (e.statusCode === 404) { + return {}; + } + throw e; } +} +async function getDataStreamCreatedOn(esClient: ElasticsearchClient, dataStream: string) { const indexSettings = await dataStreamService.getDataSteamIndexSettings(esClient, dataStream); const indexesList = Object.values(indexSettings); - const indexCreationDate = indexesList + return indexesList .map((index) => Number(index.settings?.index?.creation_date)) .sort((a, b) => a - b)[0]; +} + +type TermAggregation = Record; + +const MAX_HOSTS = MAX_HOSTS_METRIC_VALUE + 1; // Adding 1 so that we can show e.g. '50+' + +// Gather service.name terms +const serviceNamesAgg: TermAggregation = { + ['service.name']: { terms: { field: 'service.name', size: MAX_HOSTS } }, +}; + +// Gather host terms like 'host', 'pod', 'container' +const hostsAgg: TermAggregation = inventoryModels + .map((model) => findInventoryFields(model.id as InventoryItemType)) + .reduce( + (acc, fields) => ({ ...acc, [fields.id]: { terms: { field: fields.id, size: MAX_HOSTS } } }), + {} as TermAggregation + ); + +async function getDataStreamSummaryStats( + esClient: ElasticsearchClient, + dataStream: string, + start: number, + end: number +): Promise<{ + docsCount: number; + degradedDocsCount: number; + services: Record; + hosts: Record; +}> { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const response = await datasetQualityESClient.search({ + index: dataStream, + query: rangeQuery(start, end)[0], + size: 0, + aggs: { + total_count: { + value_count: { field: '_index' }, + }, + degraded_count: { + filter: { exists: { field: _IGNORED } }, + }, + ...serviceNamesAgg, + ...hostsAgg, + }, + }); + + const docsCount = Number(response.aggregations?.total_count.value ?? 0); + const degradedDocsCount = Number(response.aggregations?.degraded_count.doc_count ?? 0); return { - createdOn: indexCreationDate, + docsCount, + degradedDocsCount, + services: getTermsFromAgg(serviceNamesAgg, response.aggregations), + hosts: getTermsFromAgg(hostsAgg, response.aggregations), }; } + +async function getAvgDocSizeInBytes(esClient: ElasticsearchClient, index: string) { + const indexStats = await esClient.indices.stats({ index }); + const docCount = indexStats._all.total?.docs?.count ?? 0; + const sizeInBytes = indexStats._all.total?.store?.size_in_bytes ?? 0; + + return docCount ? sizeInBytes / docCount : 0; +} + +function getTermsFromAgg(termAgg: TermAggregation, aggregations: any) { + return Object.entries(termAgg).reduce((acc, [key, _value]) => { + const values = aggregations[key]?.buckets.map((bucket: any) => bucket.key) as string[]; + return { ...acc, [key]: values }; + }, {}); +} + +function throwIfInvalidDataStreamParams(dataStream?: string) { + if (!dataStream?.trim()) { + throw badRequest(`Data Stream name cannot be empty. Received value "${dataStream}"`); + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index a8475ed8cc8efe..6dd8590c91f0b9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -7,16 +7,17 @@ import * as t from 'io-ts'; import { keyBy, merge, values } from 'lodash'; -import { DataStreamType } from '../../../common/types'; import { DataStreamDetails, DataStreamsEstimatedDataInBytes, + DataStreamSettings, DataStreamStat, DegradedDocs, } from '../../../common/api_types'; +import { indexNameToDataStreamParts } from '../../../common/utils'; import { rangeRt, typeRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; -import { getDataStreamDetails } from './get_data_stream_details'; +import { getDataStreamDetails, getDataStreamSettings } from './get_data_stream_details'; import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; @@ -94,37 +95,71 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ }, }); +const dataStreamSettingsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const { dataStream } = params.path; + const coreContext = await context.core; + + // Query datastreams as the current user as the Kibana internal user may not have all the required permissions + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const dataStreamSettings = await getDataStreamSettings({ + esClient, + dataStream, + }); + + return dataStreamSettings; + }, +}); + const dataStreamDetailsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/details', params: t.type({ path: t.type({ dataStream: t.string, }), + query: rangeRt, }), options: { tags: [], }, async handler(resources): Promise { - const { context, params } = resources; + const { context, params, getEsCapabilities } = resources; const { dataStream } = params.path; + const { start, end } = params.query; const coreContext = await context.core; // Query datastreams as the current user as the Kibana internal user may not have all the required permissions const esClient = coreContext.elasticsearch.client.asCurrentUser; - const [type, ...datasetQuery] = dataStream.split('-'); + const { type, dataset, namespace } = indexNameToDataStreamParts(dataStream); + const sizeStatsAvailable = !(await getEsCapabilities()).serverless; const [dataStreamsStats, dataStreamDetails] = await Promise.all([ getDataStreamsStats({ esClient, - type: type as DataStreamType, - datasetQuery: datasetQuery.join('-'), + type, + datasetQuery: `${dataset}-${namespace}`, }), - getDataStreamDetails({ esClient, dataStream }), + getDataStreamDetails({ esClient, dataStream, start, end, sizeStatsAvailable }), ]); return { - createdOn: dataStreamDetails?.createdOn, + docsCount: dataStreamDetails?.docsCount, + degradedDocsCount: dataStreamDetails?.degradedDocsCount, + services: dataStreamDetails?.services, + hosts: dataStreamDetails?.hosts, + sizeBytes: dataStreamDetails?.sizeBytes, lastActivity: dataStreamsStats.items?.[0]?.lastActivity, }; }, @@ -166,5 +201,6 @@ export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, ...dataStreamDetailsRoute, + ...dataStreamSettingsRoute, ...estimatedDataInBytesRoute, }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json b/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json index 461c69492c8521..58b8cfaf987e28 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json +++ b/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json @@ -26,7 +26,6 @@ "@kbn/shared-ux-utility", "@kbn/ui-theme", "@kbn/core-notifications-browser", - "@kbn/formatters", "@kbn/data-service", "@kbn/observability-shared-plugin", "@kbn/data-plugin", @@ -42,7 +41,8 @@ "@kbn/deeplinks-management", "@kbn/deeplinks-analytics", "@kbn/core-elasticsearch-server", - "@kbn/ui-actions-plugin" + "@kbn/ui-actions-plugin", + "@kbn/metrics-data-access-plugin" ], "exclude": ["target/**/*"] } diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts index 2c809802e795ea..efa183d2d336b9 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_details.spec.ts @@ -10,18 +10,19 @@ import expect from '@kbn/expect'; import { DatasetQualityApiClientKey } from '../../common/config'; import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { expectToReject, getDataStreamSettingsOfFirstIndex } from '../../utils'; +import { expectToReject } from '../../utils'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); const synthtrace = getService('logSynthtraceEsClient'); - const esClient = getService('es'); const datasetQualityApiClient = getService('datasetQualityApiClient'); const start = '2023-12-11T18:00:00.000Z'; const end = '2023-12-11T18:01:00.000Z'; const type = 'logs'; const dataset = 'nginx.access'; const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { return await datasetQualityApiClient[user]({ @@ -30,6 +31,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { path: { dataStream, }, + query: { + start, + end, + }, }, }); } @@ -50,6 +55,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { .namespace(namespace) .defaults({ 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, }) ), ]); @@ -71,13 +78,16 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(resp.body).empty(); }); - it('returns data stream details correctly', async () => { - const dataStreamSettings = await getDataStreamSettingsOfFirstIndex( - esClient, - `logs-${dataset}-${namespace}` - ); + it('returns "sizeBytes" correctly', async () => { + const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); + expect(isNaN(resp.body.sizeBytes as number)).to.be(false); + expect(resp.body.sizeBytes).to.be.greaterThan(0); + }); + + it('returns service.name and host.name correctly', async () => { const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); - expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + expect(resp.body.services).to.eql({ ['service.name']: [serviceName] }); + expect(resp.body.hosts?.['host.name']).to.eql([hostName]); }); after(async () => { diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts new file mode 100644 index 00000000000000..e6f75d53b6c778 --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/data_stream_settings.spec.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + expectToReject, + getDataStreamSettingsOfEarliestIndex, + rolloverDataStream, +} from '../../utils'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const synthtrace = getService('logSynthtraceEsClient'); + const esClient = getService('es'); + const datasetQualityApiClient = getService('datasetQualityApiClient'); + const start = '2023-12-11T18:00:00.000Z'; + const end = '2023-12-11T18:01:00.000Z'; + const type = 'logs'; + const dataset = 'nginx.access'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + + async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { + return await datasetQualityApiClient[user]({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', + params: { + path: { + dataStream, + }, + }, + }); + } + + registry.when('DataStream Settings', { config: 'basic' }, () => { + describe('gets the data stream settings', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + it('returns error when dataStream param is not provided', async () => { + const expectedMessage = 'Data Stream name cannot be empty'; + const err = await expectToReject(() => + callApiAs('datasetQualityLogsUser', encodeURIComponent(' ')) + ); + expect(err.res.status).to.be(400); + expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); + }); + + it('returns {} if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApiAs('datasetQualityLogsUser', nonExistentDataStream); + expect(resp.body).empty(); + }); + + it('returns "createdOn" correctly', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + `${type}-${dataset}-${namespace}` + ); + const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + }); + + it('returns "createdOn" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + `${type}-${dataset}-${namespace}` + ); + const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); + }); +} diff --git a/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts b/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts index ff3a3cc95d1ac2..bdf5187db07250 100644 --- a/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts +++ b/x-pack/test/dataset_quality_api_integration/utils/data_stream.ts @@ -7,7 +7,20 @@ import { Client } from '@elastic/elasticsearch'; -export async function getDataStreamSettingsOfFirstIndex(es: Client, name: string) { +export async function rolloverDataStream(es: Client, name: string) { + return es.indices.rollover({ alias: name }); +} + +export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: string) { const matchingIndexesObj = await es.indices.getSettings({ index: name }); - return Object.values(matchingIndexesObj ?? {})[0]?.settings; + + const matchingIndexes = Object.keys(matchingIndexesObj ?? {}); + matchingIndexes.sort((a, b) => { + return ( + Number(matchingIndexesObj[a].settings?.index?.creation_date) - + Number(matchingIndexesObj[b].settings?.index?.creation_date) + ); + }); + + return matchingIndexesObj[matchingIndexes[0]].settings; } diff --git a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts index 3c9b7596feb644..68d96070990f06 100644 --- a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts +++ b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts @@ -28,12 +28,14 @@ export function getLogsForDataset({ count = 1, isMalformed = false, namespace = defaultNamespace, + services, }: { dataset: string; to: moment.MomentInput; count?: number; isMalformed?: boolean; namespace?: string; + services?: string[]; }) { return timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') @@ -46,7 +48,9 @@ export function getLogsForDataset({ timestamp, dataset, MESSAGE_LOG_LEVELS[index % MESSAGE_LOG_LEVELS.length], - SERVICE_NAMES[index % SERVICE_NAMES.length], + services?.[index] ?? + services?.[index % services.length] ?? + SERVICE_NAMES[index % SERVICE_NAMES.length], CLUSTER[index % CLUSTER.length], CLOUD_PROVIDERS[index % CLOUD_PROVIDERS.length], CLOUD_REGION[index % CLOUD_REGION.length], @@ -108,7 +112,7 @@ export function createLogRecord( cloudProvider: string, cloudRegion: string, isMalformed = false, - namespace = defaultNamespace + namespace: string = defaultNamespace ): ReturnType { return log .create() diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts index d5b2f0facfb37e..a0daafef466b70 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts @@ -144,6 +144,139 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expect(datasetSelectorText).to.eql(testDatasetName); }); + it('shows summary KPIs', async () => { + await PageObjects.datasetQuality.navigateTo(); + + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summary = await PageObjects.datasetQuality.parseFlyoutKpis(); + expect(summary).to.eql({ + docsCountTotal: '0', + size: '0.0 B', + services: '0', + hosts: '0', + degradedDocs: '0', + }); + }); + + it('shows the updated KPIs', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis(); + + // Set time range to 3 days ago + const flyoutBodyContainer = await testSubjects.find( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutBody + ); + await PageObjects.datasetQuality.setDatePickerLastXUnits(flyoutBodyContainer, 3, 'd'); + + // Index 2 doc 2 days ago + const time2DaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + }) + ); + + // Index 5 degraded docs 2 days ago + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 5, + dataset: apacheAccessDatasetName, + isMalformed: true, + }) + ); + + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis(); + + expect(parseInt(summaryAfter.docsCountTotal, 10)).to.be.greaterThan( + parseInt(summaryBefore.docsCountTotal, 10) + ); + + expect(parseInt(summaryAfter.degradedDocs, 10)).to.be.greaterThan( + parseInt(summaryBefore.degradedDocs, 10) + ); + + expect(parseInt(summaryAfter.size, 10)).to.be.greaterThan(parseInt(summaryBefore.size, 10)); + expect(parseInt(summaryAfter.services, 10)).to.be.greaterThan( + parseInt(summaryBefore.services, 10) + ); + expect(parseInt(summaryAfter.hosts, 10)).to.be.greaterThan(parseInt(summaryBefore.hosts, 10)); + }); + + it('shows the right number of services', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis(); + const testServices = ['test-srv-1', 'test-srv-2']; + + // Index 2 docs with different services + const timeNow = Date.now(); + await synthtrace.index( + getLogsForDataset({ + to: timeNow, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + services: testServices, + }) + ); + + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis(); + + expect(parseInt(summaryAfter.services, 10)).to.eql( + parseInt(summaryBefore.services, 10) + testServices.length + ); + }); + + it('goes to log explorer for degraded docs when show all is clicked', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const degradedDocsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.degradedDocs}`; + await testSubjects.click(degradedDocsShowAllSelector); + await browser.switchTab(1); + + // Confirm dataset selector text in observability logs explorer + const datasetSelectorText = + await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); + expect(datasetSelectorText).to.contain(apacheAccessDatasetName); + + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); + + it('goes to infra hosts for hosts when show all is clicked', async () => { + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const hostsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.hosts}`; + await testSubjects.click(hostsShowAllSelector); + await browser.switchTab(1); + + // Confirm url contains metrics/hosts + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + expect(parsedUrl.pathname).to.contain('/app/metrics/hosts'); + }); + + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); + it('Integration actions menu is present with correct actions', async () => { const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_summary.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_summary.ts index 1a072e622f62cb..c3bc9ea3145a57 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_summary.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_summary.ts @@ -38,7 +38,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid datasetHealthDegraded: '0', datasetHealthGood: '3', activeDatasets: '0 of 3', - estimatedData: '0 Bytes', + estimatedData: '0.0 B', }); }); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index cd59faf3c90530..39c361c834b1fe 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -34,6 +34,8 @@ type SummaryPanelKpi = Record< string >; +type FlyoutKpi = Record<'docsCountTotal' | 'size' | 'services' | 'hosts' | 'degradedDocs', string>; + export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProviderContext) { const PageObjects = getPageObjects(['common']); const testSubjects = getService('testSubjects'); @@ -69,6 +71,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv datasetQualityNamespacesSelectable: 'datasetQualityNamespacesSelectable', datasetQualityNamespacesSelectableButton: 'datasetQualityNamespacesSelectableButton', datasetQualityDatasetHealthKpi: 'datasetQualityDatasetHealthKpi', + datasetQualityFlyoutKpiValue: 'datasetQualityFlyoutKpiValue', + datasetQualityFlyoutKpiLink: 'datasetQualityFlyoutKpiLink', superDatePickerToggleQuickMenuButton: 'superDatePickerToggleQuickMenuButton', superDatePickerApplyTimeButton: 'superDatePickerApplyTimeButton', @@ -112,7 +116,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async waitUntilTableLoaded() { - await find.waitForDeletedByCssSelector('.euiBasicTable-loading'); + await find.waitForDeletedByCssSelector('.euiBasicTable-loading', 20 * 1000); }, async waitUntilSummaryPanelLoaded() { @@ -235,6 +239,16 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv return testSubjects.click(testSubjectSelectors.euiFlyoutCloseButton); }, + async refreshFlyout() { + const flyoutContainer: WebElementWrapper = await testSubjects.find( + testSubjectSelectors.datasetQualityFlyoutBody + ); + const refreshButton = await flyoutContainer.findByTestSubject( + testSubjectSelectors.superDatePickerApplyTimeButton + ); + return refreshButton.click(); + }, + async getFlyoutElementsByText(selector: string, text: string) { const flyoutContainer: WebElementWrapper = await testSubjects.find( testSubjectSelectors.datasetQualityFlyout @@ -270,12 +284,46 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv return elements.length > 0; }, + // `excludeKeys` needed to circumvent `_stats` not available in Serverless https://github.com/elastic/kibana/issues/178954 + // TODO: Remove `excludeKeys` when `_stats` is available in Serverless + async parseFlyoutKpis(excludeKeys: string[] = []): Promise { + const kpiTitleAndKeys = [ + { title: texts.docsCountTotal, key: 'docsCountTotal' }, + { title: texts.size, key: 'size' }, + { title: texts.services, key: 'services' }, + { title: texts.hosts, key: 'hosts' }, + { title: texts.degradedDocs, key: 'degradedDocs' }, + ].filter((item) => !excludeKeys.includes(item.key)); + + const kpiTexts = await Promise.all( + kpiTitleAndKeys.map(async ({ title, key }) => ({ + key, + value: await testSubjects.getVisibleText( + `${testSubjectSelectors.datasetQualityFlyoutKpiValue}-${title}` + ), + })) + ); + + return kpiTexts.reduce( + (acc, { key, value }) => ({ + ...acc, + [key]: value, + }), + {} as FlyoutKpi + ); + }, + async setDatePickerLastXUnits( container: WebElementWrapper, timeValue: number, unit: TimeUnitId ) { - await testSubjects.click(testSubjectSelectors.superDatePickerToggleQuickMenuButton); + // Only click the menu button found under the provided container + const datePickerToggleQuickMenuButton = await container.findByTestSubject( + testSubjectSelectors.superDatePickerToggleQuickMenuButton + ); + await datePickerToggleQuickMenuButton.click(); + const datePickerQuickMenu = await testSubjects.find( testSubjectSelectors.superDatePickerQuickMenu ); @@ -300,7 +348,9 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv await timeUnitSelect.focus(); await timeUnitSelect.type(unit); - (await datePickerQuickMenu.findByCssSelector(selectors.superDatePickerApplyButton)).click(); + await ( + await datePickerQuickMenu.findByCssSelector(selectors.superDatePickerApplyButton) + ).click(); return testSubjects.missingOrFail(testSubjectSelectors.superDatePickerQuickMenu); }, @@ -433,4 +483,9 @@ const texts = { datasetHealthGood: 'Good', activeDatasets: 'Active Datasets', estimatedData: 'Estimated Data', + docsCountTotal: 'Docs count (total)', + size: 'Size', + services: 'Services', + hosts: 'Hosts', + degradedDocs: 'Degraded docs', }; diff --git a/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts b/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts index def29fb09096c2..9aafefdff51ce9 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts @@ -11,7 +11,7 @@ import type { JobParamsCSV } from '@kbn/reporting-export-types-csv-common'; import type { Filter } from '@kbn/es-query'; import { FtrProviderContext } from '../../../ftr_provider_context'; -export default ({ getService }: FtrProviderContext) => { +export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); const reportingAPI = getService('svlReportingApi'); @@ -738,4 +738,4 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); -}; +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/config.ts b/x-pack/test_serverless/api_integration/test_suites/observability/config.ts index 453a3b428ceffa..792f1a766c1cd6 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/config.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/config.ts @@ -6,7 +6,8 @@ */ import { createTestConfig } from '../../config.base'; -import { services } from './apm_api_integration/common/services'; +import { services as apmServices } from './apm_api_integration/common/services'; +import { services as datasetQualityServices } from './dataset_quality_api_integration/common/services'; export default createTestConfig({ serverlessProject: 'oblt', @@ -15,7 +16,7 @@ export default createTestConfig({ reportName: 'Serverless Observability API Integration Tests', }, suiteTags: { exclude: ['skipSvlOblt'] }, - services, + services: { ...apmServices, ...datasetQualityServices }, // include settings from project controller // https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/elasticsearch.yml diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/dataset_quality_api_supertest.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/dataset_quality_api_supertest.ts new file mode 100644 index 00000000000000..2cdb6ec4fd7652 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/dataset_quality_api_supertest.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { format } from 'url'; +import supertest from 'supertest'; +import request from 'superagent'; +import type { APIClientRequestParamsOf, APIReturnType } from '@kbn/dataset-quality-plugin/common'; +import { Config, kbnTestConfig, kibanaTestSuperuserServerless } from '@kbn/test'; +import type { APIEndpoint } from '@kbn/dataset-quality-plugin/server/routes'; +import { formatRequest } from '@kbn/server-route-repository'; +import { InheritedFtrProviderContext } from '../../../../services'; + +export function createDatasetQualityApiClient(st: supertest.SuperTest) { + return async ( + options: { + type?: 'form-data'; + endpoint: TEndpoint; + } & APIClientRequestParamsOf & { params?: { query?: { _inspect?: boolean } } } + ): Promise> => { + const { endpoint, type } = options; + + const params = 'params' in options ? (options.params as Record) : {}; + + const { method, pathname, version } = formatRequest(endpoint, params.path); + const url = format({ pathname, query: params?.query }); + + const headers: Record = { + 'kbn-xsrf': 'foo', + 'x-elastic-internal-origin': 'foo', + }; + + if (version) { + headers['Elastic-Api-Version'] = version; + } + + let res: request.Response; + if (type === 'form-data') { + const fields: Array<[string, any]> = Object.entries(params.body); + const formDataRequest = st[method](url) + .set(headers) + .set('Content-type', 'multipart/form-data'); + + for (const field of fields) { + formDataRequest.field(field[0], field[1]); + } + + res = await formDataRequest; + } else if (params.body) { + res = await st[method](url).send(params.body).set(headers); + } else { + res = await st[method](url).set(headers); + } + + // supertest doesn't throw on http errors + if (res?.status !== 200) { + throw new DatasetQualityApiError(res, endpoint); + } + + return res; + }; +} + +type ApiErrorResponse = Omit & { + body: { + statusCode: number; + error: string; + message: string; + attributes: object; + }; +}; + +export type DatasetQualityApiSupertest = ReturnType; + +export class DatasetQualityApiError extends Error { + res: ApiErrorResponse; + + constructor(res: request.Response, endpoint: string) { + super( + `Unhandled DatasetQualityApiError. + Status: "${res.status}" + Endpoint: "${endpoint}" + Body: ${JSON.stringify(res.body)} + ` + ); + + this.res = res; + } +} + +async function getDatasetQualityApiClient({ svlSharedConfig }: { svlSharedConfig: Config }) { + const kibanaServer = svlSharedConfig.get('servers.kibana'); + const cAuthorities = svlSharedConfig.get('servers.kibana.certificateAuthorities'); + + const username = kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).username; + const password = kbnTestConfig.getUrlParts(kibanaTestSuperuserServerless).password; + + const url = format({ + ...kibanaServer, + auth: `${username}:${password}`, + }); + + return createDatasetQualityApiClient(supertest.agent(url, { ca: cAuthorities })); +} + +export interface SupertestReturnType { + status: number; + body: APIReturnType; +} + +type DatasetQualityApiClientKey = 'slsUser'; +export type DatasetQualityApiClient = Record< + DatasetQualityApiClientKey, + Awaited> +>; + +export async function getDatasetQualityApiClientService({ + getService, +}: InheritedFtrProviderContext): Promise { + const svlSharedConfig = getService('config'); + + return { + slsUser: await getDatasetQualityApiClient({ + svlSharedConfig, + }), + }; +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/services.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/services.ts new file mode 100644 index 00000000000000..3c15e7fffae044 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/common/services.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createLogger, LogLevel, LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { GenericFtrProviderContext } from '@kbn/test'; +import { + DatasetQualityApiClient, + getDatasetQualityApiClientService, +} from './dataset_quality_api_supertest'; +import { + InheritedServices, + InheritedFtrProviderContext, + services as inheritedServices, +} from '../../../../services'; + +export type DatasetQualityServices = InheritedServices & { + datasetQualityApiClient: ( + context: InheritedFtrProviderContext + ) => Promise; + logSynthtraceEsClient: (context: InheritedFtrProviderContext) => Promise; +}; + +export const services: DatasetQualityServices = { + ...inheritedServices, + datasetQualityApiClient: getDatasetQualityApiClientService, + logSynthtraceEsClient: async (context: InheritedFtrProviderContext) => + new LogsSynthtraceEsClient({ + client: context.getService('es'), + logger: createLogger(LogLevel.info), + refreshAfterIndex: true, + }), +}; + +export type DatasetQualityFtrContextProvider = GenericFtrProviderContext; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_details.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_details.ts new file mode 100644 index 00000000000000..1a4db10f140dc3 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_details.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { expectToReject } from './utils'; +import { + DatasetQualityApiClient, + DatasetQualityApiError, +} from './common/dataset_quality_api_supertest'; +import { DatasetQualityFtrContextProvider } from './common/services'; + +export default function ({ getService }: DatasetQualityFtrContextProvider) { + const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient'); + const synthtrace = getService('logSynthtraceEsClient'); + const start = '2023-12-11T18:00:00.000Z'; + const end = '2023-12-11T18:01:00.000Z'; + const type = 'logs'; + const dataset = 'nginx.access'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + + async function callApi(dataStream: string) { + return await datasetQualityApiClient.slsUser({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/details', + params: { + path: { + dataStream, + }, + query: { + start, + end, + }, + }, + }); + } + + describe('gets the data stream details', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + it('returns error when dataStream param is not provided', async () => { + const expectedMessage = 'Data Stream name cannot be empty'; + const err = await expectToReject(() => + callApi(encodeURIComponent(' ')) + ); + expect(err.res.status).to.be(400); + expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); + }); + + it('returns {} if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApi(nonExistentDataStream); + expect(resp.body).empty(); + }); + + it('returns "sizeBytes" as null in serverless', async () => { + const resp = await callApi(`${type}-${dataset}-${namespace}`); + expect(resp.body.sizeBytes).to.be(null); + }); + + it('returns service.name and host.name correctly', async () => { + const resp = await callApi(`${type}-${dataset}-${namespace}`); + expect(resp.body.services).to.eql({ ['service.name']: [serviceName] }); + expect(resp.body.hosts?.['host.name']).to.eql([hostName]); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts new file mode 100644 index 00000000000000..a0ea813a839312 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/data_stream_settings.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { expectToReject, getDataStreamSettingsOfEarliestIndex, rolloverDataStream } from './utils'; +import { + DatasetQualityApiClient, + DatasetQualityApiError, +} from './common/dataset_quality_api_supertest'; +import { DatasetQualityFtrContextProvider } from './common/services'; + +export default function ({ getService }: DatasetQualityFtrContextProvider) { + const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient'); + const synthtrace = getService('logSynthtraceEsClient'); + const esClient = getService('es'); + const start = '2023-12-11T18:00:00.000Z'; + const end = '2023-12-11T18:01:00.000Z'; + const type = 'logs'; + const dataset = 'nginx.access'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + + async function callApi(dataStream: string) { + return await datasetQualityApiClient.slsUser({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', + params: { + path: { + dataStream, + }, + }, + }); + } + + describe('gets the data stream settings', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + it('returns error when dataStream param is not provided', async () => { + const expectedMessage = 'Data Stream name cannot be empty'; + const err = await expectToReject(() => + callApi(encodeURIComponent(' ')) + ); + expect(err.res.status).to.be(400); + expect(err.res.body.message.indexOf(expectedMessage)).to.greaterThan(-1); + }); + + it('returns {} if matching data stream is not available', async () => { + const nonExistentDataSet = 'Non-existent'; + const nonExistentDataStream = `${type}-${nonExistentDataSet}-${namespace}`; + const resp = await callApi(nonExistentDataStream); + expect(resp.body).empty(); + }); + + it('returns "createdOn" correctly', async () => { + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + `${type}-${dataset}-${namespace}` + ); + const resp = await callApi(`${type}-${dataset}-${namespace}`); + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + }); + + it('returns "createdOn" correctly for rolled over dataStream', async () => { + await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`); + const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex( + esClient, + `${type}-${dataset}-${namespace}` + ); + const resp = await callApi(`${type}-${dataset}-${namespace}`); + expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date)); + }); + + after(async () => { + await synthtrace.clean(); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts new file mode 100644 index 00000000000000..f4022fd0098336 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Dataset Quality', function () { + loadTestFile(require.resolve('./data_stream_details')); + loadTestFile(require.resolve('./data_stream_settings')); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts new file mode 100644 index 00000000000000..bdf5187db07250 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/data_stream.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; + +export async function rolloverDataStream(es: Client, name: string) { + return es.indices.rollover({ alias: name }); +} + +export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: string) { + const matchingIndexesObj = await es.indices.getSettings({ index: name }); + + const matchingIndexes = Object.keys(matchingIndexesObj ?? {}); + matchingIndexes.sort((a, b) => { + return ( + Number(matchingIndexesObj[a].settings?.index?.creation_date) - + Number(matchingIndexesObj[b].settings?.index?.creation_date) + ); + }); + + return matchingIndexesObj[matchingIndexes[0]].settings; +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/expect_to_reject.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/expect_to_reject.ts new file mode 100644 index 00000000000000..ae352c31d71a28 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/expect_to_reject.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export async function expectToReject(fn: () => Promise): Promise { + let res: any; + try { + res = await fn(); + } catch (e) { + return e; + } + + throw new Error(`expectToReject resolved: "${JSON.stringify(res)}"`); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/index.ts new file mode 100644 index 00000000000000..0f273a5dddd100 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { expectToReject } from './expect_to_reject'; +export * from './data_stream'; diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/index.ts b/x-pack/test_serverless/api_integration/test_suites/observability/index.ts index acb33e0b4901b1..0d4f4ffe528142 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/index.ts @@ -19,5 +19,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./es_query_rule/es_query_rule')); loadTestFile(require.resolve('./slos')); loadTestFile(require.resolve('./synthetics')); + loadTestFile(require.resolve('./dataset_quality_api_integration')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts index 0d635bcb2b0e49..68d96070990f06 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts @@ -28,12 +28,14 @@ export function getLogsForDataset({ count = 1, isMalformed = false, namespace = defaultNamespace, + services, }: { dataset: string; to: moment.MomentInput; count?: number; isMalformed?: boolean; namespace?: string; + services?: string[]; }) { return timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') @@ -46,7 +48,9 @@ export function getLogsForDataset({ timestamp, dataset, MESSAGE_LOG_LEVELS[index % MESSAGE_LOG_LEVELS.length], - SERVICE_NAMES[index % SERVICE_NAMES.length], + services?.[index] ?? + services?.[index % services.length] ?? + SERVICE_NAMES[index % SERVICE_NAMES.length], CLUSTER[index % CLUSTER.length], CLOUD_PROVIDERS[index % CLOUD_PROVIDERS.length], CLOUD_REGION[index % CLOUD_REGION.length], diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts index cee287b9fa0206..fa5243ec03ef76 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts @@ -29,9 +29,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const browser = getService('browser'); const to = '2024-01-01T12:00:00.000Z'; + const excludeKeysFromServerless = ['size']; // https://github.com/elastic/kibana/issues/178954 - // FLAKY: https://github.com/elastic/kibana/issues/180994 - describe.skip('Dataset quality flyout', () => { + describe('Dataset quality flyout', () => { before(async () => { await PageObjects.svlCommonPage.loginWithRole('admin'); await synthtrace.index(getInitialTestLogs({ to, count: 4 })); @@ -85,7 +85,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(lastActivityTextExists).to.eql(true); }); - it('reflects the breakdown field state in url', async () => { + // FLAKY: https://github.com/elastic/kibana/issues/180994 + it.skip('reflects the breakdown field state in url', async () => { const testDatasetName = datasetNames[0]; await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); @@ -149,6 +150,149 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(datasetSelectorText).to.eql(testDatasetName); }); + it('shows summary KPIs', async () => { + await PageObjects.datasetQuality.navigateTo(); + + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summary = await PageObjects.datasetQuality.parseFlyoutKpis(excludeKeysFromServerless); + expect(summary).to.eql({ + docsCountTotal: '0', + // size: '0.0 B', // `_stats` not available on Serverless + services: '0', + hosts: '0', + degradedDocs: '0', + }); + }); + + it('shows the updated KPIs', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); + + // Set time range to 3 days ago + const flyoutBodyContainer = await testSubjects.find( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutBody + ); + await PageObjects.datasetQuality.setDatePickerLastXUnits(flyoutBodyContainer, 3, 'd'); + + // Index 2 doc 2 days ago + const time2DaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + }) + ); + + // Index 5 degraded docs 2 days ago + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 5, + dataset: apacheAccessDatasetName, + isMalformed: true, + }) + ); + + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); + + expect(parseInt(summaryAfter.docsCountTotal, 10)).to.be.greaterThan( + parseInt(summaryBefore.docsCountTotal, 10) + ); + + expect(parseInt(summaryAfter.degradedDocs, 10)).to.be.greaterThan( + parseInt(summaryBefore.degradedDocs, 10) + ); + + // `_stats` not available on Serverless so we can't compare size // https://github.com/elastic/kibana/issues/178954 + // expect(parseInt(summaryAfter.size, 10)).to.be.greaterThan(parseInt(summaryBefore.size, 10)); + + expect(parseInt(summaryAfter.services, 10)).to.be.greaterThan( + parseInt(summaryBefore.services, 10) + ); + expect(parseInt(summaryAfter.hosts, 10)).to.be.greaterThan(parseInt(summaryBefore.hosts, 10)); + }); + + it('shows the right number of services', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); + const testServices = ['test-srv-1', 'test-srv-2']; + + // Index 2 docs with different services + const timeNow = Date.now(); + await synthtrace.index( + getLogsForDataset({ + to: timeNow, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + services: testServices, + }) + ); + + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); + + expect(parseInt(summaryAfter.services, 10)).to.eql( + parseInt(summaryBefore.services, 10) + testServices.length + ); + }); + + it('goes to log explorer for degraded docs when show all is clicked', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const degradedDocsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.degradedDocs}`; + await testSubjects.click(degradedDocsShowAllSelector); + await browser.switchTab(1); + + // Confirm dataset selector text in observability logs explorer + const datasetSelectorText = + await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); + expect(datasetSelectorText).to.contain(apacheAccessDatasetName); + + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); + + it('goes to infra hosts for hosts when show all is clicked', async () => { + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const hostsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.hosts}`; + await testSubjects.click(hostsShowAllSelector); + await browser.switchTab(1); + + // Confirm url contains metrics/hosts + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + expect(parsedUrl.pathname).to.contain('/app/metrics/hosts'); + }); + + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); + it('Integration actions menu is present with correct actions', async () => { const apacheAccessDatasetName = 'apache.access'; const apacheAccessDatasetHumanName = 'Apache access logs'; diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_summary.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_summary.ts index 702b55c263a5c6..f5a6fd31a25f3b 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_summary.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_summary.ts @@ -40,7 +40,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { datasetHealthDegraded: '0', datasetHealthGood: '3', activeDatasets: '0 of 3', - // estimatedData: '0 Bytes', https://github.com/elastic/kibana/issues/178954 + // estimatedData: '0.0 B', https://github.com/elastic/kibana/issues/178954 }); }); @@ -131,7 +131,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(updatedActiveDatasets).to.eql('3 of 3'); - // TODO: Investigate. This fails on Serverless. + // TODO: `_stats` not available on Serverless. // https://github.com/elastic/kibana/issues/178954 // expect(_updatedEstimatedData).to.not.eql(_existingEstimatedData); }); }); diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index 21f7afafd3e2ef..1df208cc8b81bd 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -98,5 +98,6 @@ "@kbn/es-query", "@kbn/utility-types", "@kbn/synthetics-plugin", + "@kbn/dataset-quality-plugin" ] } From 53fa8bc87124711c158dbe810855639f8ac1099c Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 24 Apr 2024 13:25:12 +0200 Subject: [PATCH 008/138] [Security Solution] Network flows query "size": 0 (#181310) ## Summary Move the `"size": 0` prop to the body in the network flow aggregation queries --- .../factory/network/dns/__mocks__/index.ts | 4 ++-- .../factory/network/dns/query.dns_network.dsl.ts | 2 +- .../factory/network/http/__mocks__/index.ts | 4 ++-- .../factory/network/http/query.http_network.dsl.ts | 2 +- .../factory/network/top_countries/__mocks__/index.ts | 4 ++-- .../top_countries/query.top_countries_network.dsl.ts | 2 +- .../factory/network/top_n_flow/__mocks__/index.ts | 8 ++++---- .../network/top_n_flow/query.top_n_flow_network.dsl.ts | 10 +++++++--- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts index 7d89aae61439ef..4e5c497c06463f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts @@ -197,8 +197,8 @@ export const formattedSearchStrategyResponse = { format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }, null, @@ -277,7 +277,7 @@ export const expectedDsl = { format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts index 122bc739c7187c..cce16d8a7e5bc8 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts @@ -131,8 +131,8 @@ export const buildDnsQuery = ({ format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts index 01ceb455b080c0..b87c20c3f2810a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts @@ -673,8 +673,8 @@ export const formattedSearchStrategyResponse = { format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }, null, @@ -742,7 +742,7 @@ export const expectedDsl = { format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts index 4128de4c2ffbe7..91b036bfcabb93 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts @@ -80,8 +80,8 @@ export const buildHttpQuery = ({ format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts index 8835a98621ea3a..13c646df5a6b74 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts @@ -104,8 +104,8 @@ export const formattedSearchStrategyResponse = { ], }, }, + size: 0, }, - size: 0, track_total_hits: false, }, null, @@ -160,7 +160,7 @@ export const expectedDsl = { ], }, }, + size: 0, }, - size: 0, track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts index 72f339fb939bcf..501e8122d50e0c 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts @@ -66,8 +66,8 @@ export const buildTopCountriesQuery = ({ filter, }, }, + size: 0, }, - size: 0, track_total_hits: false, }; return dslQuery; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts index 7df0474bb29d96..00f0924f2a80b7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts @@ -947,8 +947,8 @@ export const formattedSearchStrategyResponse: NetworkTopNFlowStrategyResponse = format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }, null, @@ -997,8 +997,8 @@ export const formattedCountStrategyResponse: NetworkTopNFlowCountStrategyRespons }, }, _source: false, + size: 0, }, - size: 0, track_total_hits: false, }, null, @@ -1098,8 +1098,8 @@ export const expectedDsl = { format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }; @@ -1137,7 +1137,7 @@ export const expectedCountDsl = { }, }, _source: false, + size: 0, }, - size: 0, track_total_hits: false, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts index 10a65e361a676a..6e8154eafff9d5 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts @@ -56,8 +56,8 @@ export const buildTopNFlowQuery = ({ format: 'strict_date_optional_time', }, ], + size: 0, }, - size: 0, track_total_hits: false, }; return dslQuery; @@ -75,8 +75,12 @@ export const buildTopNFlowCountQuery = ({ allow_no_indices: true, index: defaultIndex, ignore_unavailable: true, - body: { aggregations: getCountAgg(flowTarget), query, _source: false }, - size: 0, + body: { + aggregations: getCountAgg(flowTarget), + query, + _source: false, + size: 0, + }, track_total_hits: false, }; return dslQuery; From d4a9132ef5e174d4ccba15dcb7800d9f5b2e5302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=BCrkan=20YALAMAN?= Date: Wed, 24 Apr 2024 13:29:13 +0200 Subject: [PATCH 009/138] [Search] Update connectors license (#181153) ## Summary Updates various connector configurations. - Box, Notion, Slack, Teams, and Zoom are set to run native. - Oracle, Outlook and Gmail are GA now. - GraphQL and OpenText Documentum added as Tech Preview connector clients. - Fixed a few tooltip issues. - Fixed a Teams `service_type` in integrations page. Note, waiting for icons for GraphQL and OpenText Documentum before merging. Screenshots below will have broken images until they are added. "Select Connector" with basic license: Screenshot 2024-04-18 at 15 33 30 Screenshot 2024-04-18 at 15 33 35 Integrations tiles: Screenshot 2024-04-18 at 14 51 28 Screenshot 2024-04-18 at 14 51 34 Native Upgrades: Screenshot 2024-04-18 at 14 50 12 Screenshot 2024-04-18 at 14 50 21 Screenshot 2024-04-18 at 14 50 32 Screenshot 2024-04-18 at 14 50 36 Screenshot 2024-04-18 at 14 50 40 ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../types/native_connectors.ts | 610 +++++++++++++++++- .../apis/custom_integration/integrations.ts | 2 +- .../search_connectors/common/connectors.ts | 57 +- .../public/assets/icons/graphql.svg | 4 + .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 7 files changed, 652 insertions(+), 24 deletions(-) create mode 100644 x-pack/plugins/search_connectors/public/assets/icons/graphql.svg diff --git a/packages/kbn-search-connectors/types/native_connectors.ts b/packages/kbn-search-connectors/types/native_connectors.ts index 6108280478e2a3..a044dd717ef4ff 100644 --- a/packages/kbn-search-connectors/types/native_connectors.ts +++ b/packages/kbn-search-connectors/types/native_connectors.ts @@ -73,13 +73,12 @@ const ENABLE_DOCUMENT_LEVEL_SECURITY_LABEL = i18n.translate( } ); -const ENABLE_DOCUMENT_LEVEL_SECURITY_TOOLTIP = i18n.translate( - 'searchConnectors.nativeConnectors.enableDLS.tooltip', - { +const getEnableDocumentLevelSecurityTooltip = (serviceName: string) => + i18n.translate('searchConnectors.nativeConnectors.enableDLS.tooltip', { defaultMessage: - 'Document level security ensures identities and permissions set in Google Drive are maintained in Elasticsearch. This enables you to restrict and personalize read-access users and groups have to documents in this index. Access control syncs ensure this metadata is kept up to date in your Elasticsearch documents.', - } -); + 'Document level security ensures identities and permissions set in {serviceName} are maintained in Elasticsearch. This enables you to restrict and personalize read-access users and groups have to documents in this index. Access control syncs ensure this metadata is kept up to date in your Elasticsearch documents.', + values: { serviceName }, + }); const DATABASE_LABEL = i18n.translate('searchConnectors.nativeConnectors.databaseLabel', { defaultMessage: 'Database', @@ -246,6 +245,177 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record c.id === 'sample_data_all')).to.be.above( diff --git a/x-pack/plugins/search_connectors/common/connectors.ts b/x-pack/plugins/search_connectors/common/connectors.ts index 9ffb1ee82254af..dd96c11487279e 100644 --- a/x-pack/plugins/search_connectors/common/connectors.ts +++ b/x-pack/plugins/search_connectors/common/connectors.ts @@ -220,6 +220,24 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ }), serviceType: 'google_drive', }, + { + categories: ['enterprise_search', 'elastic_stack', 'custom', 'connector', 'connector_client'], + description: i18n.translate( + 'searchConnectorsPlugin.content.nativeConnectors.graphQL.description', + { + defaultMessage: 'Search over your content with GraphQL.', + } + ), + iconPath: 'graphql.svg', + isBeta: false, + isNative: false, + keywords: ['graphql', 'connector'], + name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.graphQL.name', { + defaultMessage: 'GraphQL', + }), + serviceType: 'graphql', + isTechPreview: true, + }, { categories: [ 'enterprise_search', @@ -317,7 +335,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ ), iconPath: 'notion.svg', isBeta: true, - isNative: false, + isNative: true, keywords: ['notion', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.notion.name', { defaultMessage: 'Notion', @@ -452,7 +470,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ } ), iconPath: 'gmail.svg', - isBeta: true, + isBeta: false, isNative: true, keywords: ['gmail', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.gmail.name', { @@ -460,6 +478,27 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ }), serviceType: 'gmail', }, + { + categories: ['enterprise_search', 'elastic_stack', 'connector', 'connector_client'], + description: i18n.translate( + 'searchConnectorsPlugin.content.nativeConnectors.openTextDocumentum.description', + { + defaultMessage: 'Search over your content on OpenText Documentum.', + } + ), + iconPath: 'connector.svg', + isBeta: false, + isNative: false, + isTechPreview: true, + keywords: ['opentext', 'documentum', 'connector'], + name: i18n.translate( + 'searchConnectorsPlugin.content.nativeConnectors.openTextDocumentum.name', + { + defaultMessage: 'OpenText Documentum', + } + ), + serviceType: 'opentext_documentum', + }, { categories: [ 'enterprise_search', @@ -476,7 +515,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ } ), iconPath: 'oracle.svg', - isBeta: true, + isBeta: false, isNative: true, keywords: ['oracle', 'sql', 'database', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.oracle.name', { @@ -538,7 +577,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ categories: ['enterprise_search', 'elastic_stack', 'connector', 'connector_client'], iconPath: 'slack.svg', isBeta: false, - isNative: false, + isNative: true, isTechPreview: true, keywords: ['slack', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.slack.name', { @@ -578,7 +617,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ }), iconPath: 'box.svg', isBeta: false, - isNative: false, + isNative: true, isTechPreview: true, keywords: ['cloud', 'box'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.box.name', { @@ -602,7 +641,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ 'outlook', ], iconPath: 'outlook.svg', - isBeta: true, + isBeta: false, isNative: true, keywords: ['outlook', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.outlook.name', { @@ -627,13 +666,13 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ ), iconPath: 'teams.svg', isBeta: false, - isNative: false, + isNative: true, isTechPreview: true, keywords: ['teams', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.teams.name', { defaultMessage: 'Teams', }), - serviceType: 'teams', + serviceType: 'microsoft_teams', }, { categories: [ @@ -652,7 +691,7 @@ export const CONNECTOR_DEFINITIONS: ConnectorServerSideDefinition[] = [ ), iconPath: 'zoom.svg', isBeta: false, - isNative: false, + isNative: true, isTechPreview: true, keywords: ['zoom', 'connector'], name: i18n.translate('searchConnectorsPlugin.content.nativeConnectors.zoom.name', { diff --git a/x-pack/plugins/search_connectors/public/assets/icons/graphql.svg b/x-pack/plugins/search_connectors/public/assets/icons/graphql.svg new file mode 100644 index 00000000000000..99b540ae7b5a9b --- /dev/null +++ b/x-pack/plugins/search_connectors/public/assets/icons/graphql.svg @@ -0,0 +1,4 @@ + + + + diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index d466b6de05150b..38d5b7847a6263 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -5619,7 +5619,6 @@ "searchConnectors.nativeConnectors.databaseLabel": "Base de données", "searchConnectors.nativeConnectors.dropbox.name": "Dropbox", "searchConnectors.nativeConnectors.enableDLS.label": "Activer la sécurité au niveau du document", - "searchConnectors.nativeConnectors.enableDLS.tooltip": "La sécurité au niveau du document préserve dans Elasticsearch les identités et permissions paramétrées dans Google Drive. Vous pouvez ainsi restreindre et personnaliser l'accès en lecture des utilisateurs et des groupes pour les documents dans cet index. La synchronisation de contrôle d'accès garantit que ces métadonnées sont correctement actualisées dans vos documents Elasticsearch.", "searchConnectors.nativeConnectors.enableSSL.label": "Activer SSL", "searchConnectors.nativeConnectors.gdrive.label": "Compte de service JSON Google Drive", "searchConnectors.nativeConnectors.gdrive.maxHTTPRequest.label": "Requêtes HTTP simultanées maximales", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index a2b7a4293ffc79..7fe6ca3a6fdf50 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -5612,7 +5612,6 @@ "searchConnectors.nativeConnectors.databaseLabel": "データベース", "searchConnectors.nativeConnectors.dropbox.name": "Dropbox", "searchConnectors.nativeConnectors.enableDLS.label": "ドキュメントレベルのセキュリティを有効化", - "searchConnectors.nativeConnectors.enableDLS.tooltip": "ドキュメントレベルのセキュリティにより、Google Driveで設定されたIDと権限がElasticsearchでも維持されます。これにより、このインデックス内のドキュメントに対するユーザーやグループの読み取りアクセスを制限し、パーソナライズすることができます。アクセス制御の同期により、Elasticsearchドキュメント内のメタデータは常に最新の状態に保たれます。", "searchConnectors.nativeConnectors.enableSSL.label": "SSLを有効にする", "searchConnectors.nativeConnectors.gdrive.label": "Google DriveサービスアカウントJSON", "searchConnectors.nativeConnectors.gdrive.maxHTTPRequest.label": "最大同時HTTPリクエスト数", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 80c2ea8e3fe6cb..d57e2b983cd879 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -5623,7 +5623,6 @@ "searchConnectors.nativeConnectors.databaseLabel": "数据库", "searchConnectors.nativeConnectors.dropbox.name": "Dropbox", "searchConnectors.nativeConnectors.enableDLS.label": "启用文档级别安全性", - "searchConnectors.nativeConnectors.enableDLS.tooltip": "文档级别安全性确保在 Elasticsearch 中维护在 Google 云端硬盘中设置的身份和权限。这样，您就可以限制用户和组对此索引中的文档具有的读取访问权限并对其进行个性化。访问控制同步将确保此元数据在 Elasticsearch 文档中保持最新。", "searchConnectors.nativeConnectors.enableSSL.label": "启用 SSL", "searchConnectors.nativeConnectors.gdrive.label": "Google 云端硬盘服务帐户 JSON", "searchConnectors.nativeConnectors.gdrive.maxHTTPRequest.label": "最大并发 HTTP 请求数", From 060d99bd1bf5fc53c9c712d3baebd278c27929b8 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Wed, 24 Apr 2024 13:31:06 +0200 Subject: [PATCH 010/138] Use permanent cache for translation files on production (#181377) ## Summary Fix https://github.com/elastic/kibana/issues/83409 Use a permanent cache (`public, max-age=365d, immutable`) for translation files when in production (`dist`), similar to what we're doing for static assets. Translation files cache busting is a little tricky, because it doesn't only depend on the version (enabling or disabling a custom plugin can change the translations while not changing the build hash), so we're using a custom hash generated from the content of the current translation file (which was already used to generate the `etag` header previously). --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../i18n/core-i18n-server-internal/index.ts | 2 +- .../src/i18n_service.test.ts | 11 ++- .../src/i18n_service.ts | 36 +++++-- .../core-i18n-server-internal/src/index.ts | 1 + .../src/routes/index.ts | 14 ++- .../src/routes/translations.test.ts | 22 ++++- .../src/routes/translations.ts | 99 ++++++++++++------- .../src/i18n_service.mock.ts | 15 ++- .../core/i18n/core-i18n-server/src/types.ts | 5 + .../rendering_service.test.ts.snap | 32 +++--- .../src/rendering_service.tsx | 19 ++-- .../src/test_helpers/params.ts | 3 + .../src/types.ts | 4 + .../tsconfig.json | 3 + .../core-root-server-internal/src/server.ts | 9 +- .../api_integration/apis/core/translations.ts | 7 +- .../server_integration/http/platform/cache.ts | 2 +- .../test_suites/common/core/translations.ts | 7 +- 18 files changed, 209 insertions(+), 82 deletions(-) diff --git a/packages/core/i18n/core-i18n-server-internal/index.ts b/packages/core/i18n/core-i18n-server-internal/index.ts index dbb51964c10e5c..e4709a8cd7b832 100644 --- a/packages/core/i18n/core-i18n-server-internal/index.ts +++ b/packages/core/i18n/core-i18n-server-internal/index.ts @@ -6,5 +6,5 @@ * Side Public License, v 1. */ -export type { I18nConfigType } from './src'; +export type { I18nConfigType, InternalI18nServicePreboot } from './src'; export { config, I18nService } from './src'; diff --git a/packages/core/i18n/core-i18n-server-internal/src/i18n_service.test.ts b/packages/core/i18n/core-i18n-server-internal/src/i18n_service.test.ts index 88ac8daac65e7e..fe4129d8c59954 100644 --- a/packages/core/i18n/core-i18n-server-internal/src/i18n_service.test.ts +++ b/packages/core/i18n/core-i18n-server-internal/src/i18n_service.test.ts @@ -37,12 +37,13 @@ describe('I18nService', () => { let configService: ReturnType; let httpPreboot: ReturnType; let httpSetup: ReturnType; + let coreContext: ReturnType; beforeEach(() => { jest.clearAllMocks(); configService = getConfigService(); - const coreContext = mockCoreContext.create({ configService }); + coreContext = mockCoreContext.create({ configService }); service = new I18nService(coreContext); httpPreboot = httpServiceMock.createInternalPrebootContract(); @@ -73,13 +74,15 @@ describe('I18nService', () => { expect(initTranslationsMock).toHaveBeenCalledWith('en', translationFiles); }); - it('calls `registerRoutesMock` with the correct parameters', async () => { + it('calls `registerRoutes` with the correct parameters', async () => { await service.preboot({ pluginPaths: [], http: httpPreboot }); expect(registerRoutesMock).toHaveBeenCalledTimes(1); expect(registerRoutesMock).toHaveBeenCalledWith({ locale: 'en', router: expect.any(Object), + isDist: coreContext.env.packageInfo.dist, + translationHash: expect.any(String), }); }); }); @@ -114,13 +117,15 @@ describe('I18nService', () => { expect(initTranslationsMock).toHaveBeenCalledWith('en', translationFiles); }); - it('calls `registerRoutesMock` with the correct parameters', async () => { + it('calls `registerRoutes` with the correct parameters', async () => { await service.setup({ pluginPaths: [], http: httpSetup }); expect(registerRoutesMock).toHaveBeenCalledTimes(1); expect(registerRoutesMock).toHaveBeenCalledWith({ locale: 'en', router: expect.any(Object), + isDist: coreContext.env.packageInfo.dist, + translationHash: expect.any(String), }); }); diff --git a/packages/core/i18n/core-i18n-server-internal/src/i18n_service.ts b/packages/core/i18n/core-i18n-server-internal/src/i18n_service.ts index 7f482564799233..d7ff9d903680ce 100644 --- a/packages/core/i18n/core-i18n-server-internal/src/i18n_service.ts +++ b/packages/core/i18n/core-i18n-server-internal/src/i18n_service.ts @@ -7,6 +7,8 @@ */ import { firstValueFrom } from 'rxjs'; +import { createHash } from 'crypto'; +import { i18n, Translation } from '@kbn/i18n'; import type { Logger } from '@kbn/logging'; import type { IConfigService } from '@kbn/config'; import type { CoreContext } from '@kbn/core-base-server-internal'; @@ -30,29 +32,42 @@ export interface SetupDeps { pluginPaths: string[]; } +export interface InternalI18nServicePreboot { + getTranslationHash(): string; +} + export class I18nService { private readonly log: Logger; private readonly configService: IConfigService; - constructor(coreContext: CoreContext) { + constructor(private readonly coreContext: CoreContext) { this.log = coreContext.logger.get('i18n'); this.configService = coreContext.configService; } - public async preboot({ pluginPaths, http }: PrebootDeps) { - const { locale } = await this.initTranslations(pluginPaths); - http.registerRoutes('', (router) => registerRoutes({ router, locale })); + public async preboot({ pluginPaths, http }: PrebootDeps): Promise { + const { locale, translationHash } = await this.initTranslations(pluginPaths); + const { dist: isDist } = this.coreContext.env.packageInfo; + http.registerRoutes('', (router) => + registerRoutes({ router, locale, isDist, translationHash }) + ); + + return { + getTranslationHash: () => translationHash, + }; } public async setup({ pluginPaths, http }: SetupDeps): Promise { - const { locale, translationFiles } = await this.initTranslations(pluginPaths); + const { locale, translationFiles, translationHash } = await this.initTranslations(pluginPaths); const router = http.createRouter(''); - registerRoutes({ router, locale }); + const { dist: isDist } = this.coreContext.env.packageInfo; + registerRoutes({ router, locale, isDist, translationHash }); return { getLocale: () => locale, getTranslationFiles: () => translationFiles, + getTranslationHash: () => translationHash, }; } @@ -69,6 +84,13 @@ export class I18nService { this.log.debug(`Using translation files: [${translationFiles.join(', ')}]`); await initTranslations(locale, translationFiles); - return { locale, translationFiles }; + const translationHash = getTranslationHash(i18n.getTranslation()); + + return { locale, translationFiles, translationHash }; } } + +const getTranslationHash = (translations: Translation) => { + const serialized = JSON.stringify(translations); + return createHash('sha256').update(serialized).digest('hex').slice(0, 12); +}; diff --git a/packages/core/i18n/core-i18n-server-internal/src/index.ts b/packages/core/i18n/core-i18n-server-internal/src/index.ts index a87d88ec28dd70..9ef1fe5c962919 100644 --- a/packages/core/i18n/core-i18n-server-internal/src/index.ts +++ b/packages/core/i18n/core-i18n-server-internal/src/index.ts @@ -9,3 +9,4 @@ export { config } from './i18n_config'; export type { I18nConfigType } from './i18n_config'; export { I18nService } from './i18n_service'; +export type { InternalI18nServicePreboot } from './i18n_service'; diff --git a/packages/core/i18n/core-i18n-server-internal/src/routes/index.ts b/packages/core/i18n/core-i18n-server-internal/src/routes/index.ts index 09d49f2f23cab0..64d3a21fa81c5c 100644 --- a/packages/core/i18n/core-i18n-server-internal/src/routes/index.ts +++ b/packages/core/i18n/core-i18n-server-internal/src/routes/index.ts @@ -9,6 +9,16 @@ import type { IRouter } from '@kbn/core-http-server'; import { registerTranslationsRoute } from './translations'; -export const registerRoutes = ({ router, locale }: { router: IRouter; locale: string }) => { - registerTranslationsRoute(router, locale); +export const registerRoutes = ({ + router, + locale, + isDist, + translationHash, +}: { + router: IRouter; + locale: string; + isDist: boolean; + translationHash: string; +}) => { + registerTranslationsRoute({ router, locale, isDist, translationHash }); }; diff --git a/packages/core/i18n/core-i18n-server-internal/src/routes/translations.test.ts b/packages/core/i18n/core-i18n-server-internal/src/routes/translations.test.ts index 9d9f19c3815771..a0a04b16033ac7 100644 --- a/packages/core/i18n/core-i18n-server-internal/src/routes/translations.test.ts +++ b/packages/core/i18n/core-i18n-server-internal/src/routes/translations.test.ts @@ -12,11 +12,27 @@ import { registerTranslationsRoute } from './translations'; describe('registerTranslationsRoute', () => { test('registers route with expected options', () => { const router = mockRouter.create(); - registerTranslationsRoute(router, 'en'); - expect(router.get).toHaveBeenCalledTimes(1); + registerTranslationsRoute({ + router, + locale: 'en', + isDist: true, + translationHash: 'XXXX', + }); + expect(router.get).toHaveBeenCalledTimes(2); expect(router.get).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ options: { access: 'public', authRequired: false } }), + expect.objectContaining({ + path: '/translations/{locale}.json', + options: { access: 'public', authRequired: false }, + }), + expect.any(Function) + ); + expect(router.get).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + path: '/translations/XXXX/{locale}.json', + options: { access: 'public', authRequired: false }, + }), expect.any(Function) ); }); diff --git a/packages/core/i18n/core-i18n-server-internal/src/routes/translations.ts b/packages/core/i18n/core-i18n-server-internal/src/routes/translations.ts index 49a6779c7d3b2c..369bc0c6c585f7 100644 --- a/packages/core/i18n/core-i18n-server-internal/src/routes/translations.ts +++ b/packages/core/i18n/core-i18n-server-internal/src/routes/translations.ts @@ -6,54 +6,81 @@ * Side Public License, v 1. */ -import { createHash } from 'crypto'; import { i18n } from '@kbn/i18n'; import { schema } from '@kbn/config-schema'; import type { IRouter } from '@kbn/core-http-server'; +const MINUTE = 60; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; + interface TranslationCache { translations: string; hash: string; } -export const registerTranslationsRoute = (router: IRouter, locale: string) => { +export const registerTranslationsRoute = ({ + router, + locale, + translationHash, + isDist, +}: { + router: IRouter; + locale: string; + translationHash: string; + isDist: boolean; +}) => { let translationCache: TranslationCache; - router.get( - { - path: '/translations/{locale}.json', - validate: { - params: schema.object({ - locale: schema.string(), - }), - }, - options: { - access: 'public', - authRequired: false, - }, - }, - (ctx, req, res) => { - if (req.params.locale.toLowerCase() !== locale.toLowerCase()) { - return res.notFound({ - body: `Unknown locale: ${req.params.locale}`, - }); - } - if (!translationCache) { - const translations = JSON.stringify(i18n.getTranslation()); - const hash = createHash('sha1').update(translations).digest('hex'); - translationCache = { - translations, - hash, - }; - } - return res.ok({ - headers: { - 'content-type': 'application/json', - 'cache-control': 'must-revalidate', - etag: translationCache.hash, + ['/translations/{locale}.json', `/translations/${translationHash}/{locale}.json`].forEach( + (routePath) => { + router.get( + { + path: routePath, + validate: { + params: schema.object({ + locale: schema.string(), + }), + }, + options: { + access: 'public', + authRequired: false, + }, }, - body: translationCache.translations, - }); + (ctx, req, res) => { + if (req.params.locale.toLowerCase() !== locale.toLowerCase()) { + return res.notFound({ + body: `Unknown locale: ${req.params.locale}`, + }); + } + if (!translationCache) { + const translations = JSON.stringify(i18n.getTranslation()); + translationCache = { + translations, + hash: translationHash, + }; + } + + let headers: Record; + if (isDist) { + headers = { + 'content-type': 'application/json', + 'cache-control': `public, max-age=${365 * DAY}, immutable`, + }; + } else { + headers = { + 'content-type': 'application/json', + 'cache-control': 'must-revalidate', + etag: translationCache.hash, + }; + } + + return res.ok({ + headers, + body: translationCache.translations, + }); + } + ); } ); }; diff --git a/packages/core/i18n/core-i18n-server-mocks/src/i18n_service.mock.ts b/packages/core/i18n/core-i18n-server-mocks/src/i18n_service.mock.ts index 41c6dadea85b96..bc954a93089b79 100644 --- a/packages/core/i18n/core-i18n-server-mocks/src/i18n_service.mock.ts +++ b/packages/core/i18n/core-i18n-server-mocks/src/i18n_service.mock.ts @@ -7,17 +7,29 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import type { I18nService } from '@kbn/core-i18n-server-internal'; +import type { I18nService, InternalI18nServicePreboot } from '@kbn/core-i18n-server-internal'; import type { I18nServiceSetup } from '@kbn/core-i18n-server'; const createSetupContractMock = () => { const mock: jest.Mocked = { getLocale: jest.fn(), getTranslationFiles: jest.fn(), + getTranslationHash: jest.fn(), }; mock.getLocale.mockReturnValue('en'); mock.getTranslationFiles.mockReturnValue([]); + mock.getTranslationHash.mockReturnValue('MOCK_HASH'); + + return mock; +}; + +const createInternalPrebootMock = () => { + const mock: jest.Mocked = { + getTranslationHash: jest.fn(), + }; + + mock.getTranslationHash.mockReturnValue('MOCK_HASH'); return mock; }; @@ -38,4 +50,5 @@ const createMock = () => { export const i18nServiceMock = { create: createMock, createSetupContract: createSetupContractMock, + createInternalPrebootContract: createInternalPrebootMock, }; diff --git a/packages/core/i18n/core-i18n-server/src/types.ts b/packages/core/i18n/core-i18n-server/src/types.ts index 9c640612c9034f..06b72d76c46fea 100644 --- a/packages/core/i18n/core-i18n-server/src/types.ts +++ b/packages/core/i18n/core-i18n-server/src/types.ts @@ -19,4 +19,9 @@ export interface I18nServiceSetup { * Return the absolute paths to translation files currently in use. */ getTranslationFiles(): string[]; + + /** + * Returns the hash generated from the current translations. + */ + getTranslationHash(): string; } diff --git a/packages/core/rendering/core-rendering-server-internal/src/__snapshots__/rendering_service.test.ts.snap b/packages/core/rendering/core-rendering-server-internal/src/__snapshots__/rendering_service.test.ts.snap index 250073318a3ac5..69f534cf837b48 100644 --- a/packages/core/rendering/core-rendering-server-internal/src/__snapshots__/rendering_service.test.ts.snap +++ b/packages/core/rendering/core-rendering-server-internal/src/__snapshots__/rendering_service.test.ts.snap @@ -37,7 +37,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -117,7 +117,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -193,7 +193,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -273,7 +273,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -349,7 +349,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -425,7 +425,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -505,7 +505,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -581,7 +581,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -662,7 +662,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -742,7 +742,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -823,7 +823,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -908,7 +908,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -984,7 +984,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -1065,7 +1065,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -1150,7 +1150,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { @@ -1231,7 +1231,7 @@ Object { ], }, "i18n": Object { - "translationsUrl": "/mock-server-basepath/translations/en.json", + "translationsUrl": "/mock-server-basepath/translations/MOCK_HASH/en.json", }, "legacyMetadata": Object { "globalUiSettings": Object { diff --git a/packages/core/rendering/core-rendering-server-internal/src/rendering_service.tsx b/packages/core/rendering/core-rendering-server-internal/src/rendering_service.tsx index 5b8f86a96697b6..cf97bad34fc605 100644 --- a/packages/core/rendering/core-rendering-server-internal/src/rendering_service.tsx +++ b/packages/core/rendering/core-rendering-server-internal/src/rendering_service.tsx @@ -10,7 +10,7 @@ import React from 'react'; import { renderToStaticMarkup } from 'react-dom/server'; import { firstValueFrom, of } from 'rxjs'; import { catchError, take, timeout } from 'rxjs'; -import { i18n } from '@kbn/i18n'; +import { i18n as i18nLib } from '@kbn/i18n'; import type { ThemeVersion } from '@kbn/ui-shared-deps-npm'; import type { CoreContext } from '@kbn/core-base-server-internal'; @@ -61,6 +61,7 @@ export class RenderingService { public async preboot({ http, uiPlugins, + i18n, }: RenderingPrebootDeps): Promise { http.registerRoutes('', (router) => { registerBootstrapRoute({ @@ -75,7 +76,7 @@ export class RenderingService { }); return { - render: this.render.bind(this, { http, uiPlugins }), + render: this.render.bind(this, { http, uiPlugins, i18n }), }; } @@ -86,6 +87,7 @@ export class RenderingService { uiPlugins, customBranding, userSettings, + i18n, }: RenderingSetupDeps): Promise { registerBootstrapRoute({ router: http.createRouter(''), @@ -106,6 +108,7 @@ export class RenderingService { status, customBranding, userSettings, + i18n, }), }; } @@ -119,7 +122,8 @@ export class RenderingService { }, { isAnonymousPage = false, vars, includeExposedConfigKeys }: IRenderOptions = {} ) { - const { elasticsearch, http, uiPlugins, status, customBranding, userSettings } = renderOptions; + const { elasticsearch, http, uiPlugins, status, customBranding, userSettings, i18n } = + renderOptions; const env = { mode: this.coreContext.env.mode, @@ -201,14 +205,17 @@ export class RenderingService { const loggingConfig = await getBrowserLoggingConfig(this.coreContext.configService); + const translationHash = i18n.getTranslationHash(); + const translationsUrl = `${serverBasePath}/translations/${translationHash}/${i18nLib.getLocale()}.json`; + const filteredPlugins = filterUiPlugins({ uiPlugins, isAnonymousPage }); const bootstrapScript = isAnonymousPage ? 'bootstrap-anonymous.js' : 'bootstrap.js'; const metadata: RenderingMetadata = { strictCsp: http.csp.strict, uiPublicUrl: `${staticAssetsHrefBase}/ui`, bootstrapScriptUrl: `${basePath}/${bootstrapScript}`, - i18n: i18n.translate, - locale: i18n.getLocale(), + i18n: i18nLib.translate, + locale: i18nLib.getLocale(), themeVersion, darkMode, stylesheetPaths: commonStylesheetPaths, @@ -233,7 +240,7 @@ export class RenderingService { anonymousStatusPage: status?.isStatusPageAnonymous() ?? false, i18n: { // TODO: Make this load as part of static assets! - translationsUrl: `${basePath}/translations/${i18n.getLocale()}.json`, + translationsUrl, }, theme: { darkMode, diff --git a/packages/core/rendering/core-rendering-server-internal/src/test_helpers/params.ts b/packages/core/rendering/core-rendering-server-internal/src/test_helpers/params.ts index 42c48d9e0bd210..d7b44157f4a6d5 100644 --- a/packages/core/rendering/core-rendering-server-internal/src/test_helpers/params.ts +++ b/packages/core/rendering/core-rendering-server-internal/src/test_helpers/params.ts @@ -12,6 +12,7 @@ import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { statusServiceMock } from '@kbn/core-status-server-mocks'; import { customBrandingServiceMock } from '@kbn/core-custom-branding-server-mocks'; import { userSettingsServiceMock } from '@kbn/core-user-settings-server-mocks'; +import { i18nServiceMock } from '@kbn/core-i18n-server-mocks'; const context = mockCoreContext.create(); const httpPreboot = httpServiceMock.createInternalPrebootContract(); @@ -33,6 +34,7 @@ export const mockRenderingServiceParams = context; export const mockRenderingPrebootDeps = { http: httpPreboot, uiPlugins: createUiPlugins(), + i18n: i18nServiceMock.createInternalPrebootContract(), }; export const mockRenderingSetupDeps = { elasticsearch, @@ -41,4 +43,5 @@ export const mockRenderingSetupDeps = { customBranding, status, userSettings, + i18n: i18nServiceMock.createSetupContract(), }; diff --git a/packages/core/rendering/core-rendering-server-internal/src/types.ts b/packages/core/rendering/core-rendering-server-internal/src/types.ts index e96353816e199c..357f17182b2d5e 100644 --- a/packages/core/rendering/core-rendering-server-internal/src/types.ts +++ b/packages/core/rendering/core-rendering-server-internal/src/types.ts @@ -22,6 +22,8 @@ import type { UiPlugins } from '@kbn/core-plugins-base-server-internal'; import type { InternalCustomBrandingSetup } from '@kbn/core-custom-branding-server-internal'; import type { CustomBranding } from '@kbn/core-custom-branding-common'; import type { InternalUserSettingsServiceSetup } from '@kbn/core-user-settings-server-internal'; +import type { I18nServiceSetup } from '@kbn/core-i18n-server'; +import type { InternalI18nServicePreboot } from '@kbn/core-i18n-server-internal'; /** @internal */ export interface RenderingMetadata { @@ -42,6 +44,7 @@ export interface RenderingMetadata { export interface RenderingPrebootDeps { http: InternalHttpServicePreboot; uiPlugins: UiPlugins; + i18n: InternalI18nServicePreboot; } /** @internal */ @@ -52,6 +55,7 @@ export interface RenderingSetupDeps { uiPlugins: UiPlugins; customBranding: InternalCustomBrandingSetup; userSettings: InternalUserSettingsServiceSetup; + i18n: I18nServiceSetup; } /** @internal */ diff --git a/packages/core/rendering/core-rendering-server-internal/tsconfig.json b/packages/core/rendering/core-rendering-server-internal/tsconfig.json index ba9dfdd87f3077..e306dca24059c8 100644 --- a/packages/core/rendering/core-rendering-server-internal/tsconfig.json +++ b/packages/core/rendering/core-rendering-server-internal/tsconfig.json @@ -41,6 +41,9 @@ "@kbn/core-user-settings-server-internal", "@kbn/core-logging-common-internal", "@kbn/core-logging-server-internal", + "@kbn/core-i18n-server", + "@kbn/core-i18n-server-internal", + "@kbn/core-i18n-server-mocks", ], "exclude": [ "target/**/*", diff --git a/packages/core/root/core-root-server-internal/src/server.ts b/packages/core/root/core-root-server-internal/src/server.ts index 1439fa19cb64da..fc34e151e657f0 100644 --- a/packages/core/root/core-root-server-internal/src/server.ts +++ b/packages/core/root/core-root-server-internal/src/server.ts @@ -195,7 +195,7 @@ export class Server { const httpPreboot = await this.http.preboot({ context: contextServicePreboot }); // setup i18n prior to any other service, to have translations ready - await this.i18n.preboot({ http: httpPreboot, pluginPaths }); + const i18nPreboot = await this.i18n.preboot({ http: httpPreboot, pluginPaths }); this.capabilities.preboot({ http: httpPreboot }); @@ -203,7 +203,11 @@ export class Server { await this.status.preboot({ http: httpPreboot }); - const renderingPreboot = await this.rendering.preboot({ http: httpPreboot, uiPlugins }); + const renderingPreboot = await this.rendering.preboot({ + http: httpPreboot, + uiPlugins, + i18n: i18nPreboot, + }); const httpResourcesPreboot = this.httpResources.preboot({ http: httpPreboot, @@ -328,6 +332,7 @@ export class Server { uiPlugins, customBranding: customBrandingSetup, userSettings: userSettingsServiceSetup, + i18n: i18nServiceSetup, }); const httpResourcesSetup = this.httpResources.setup({ diff --git a/test/api_integration/apis/core/translations.ts b/test/api_integration/apis/core/translations.ts index 9e492556fdd964..c29cdbf28da77d 100644 --- a/test/api_integration/apis/core/translations.ts +++ b/test/api_integration/apis/core/translations.ts @@ -18,8 +18,11 @@ export default function ({ getService }: FtrProviderContext) { expect(response.body.locale).to.eql('en'); expect(response.header).to.have.property('content-type', 'application/json; charset=utf-8'); - expect(response.header).to.have.property('cache-control', 'must-revalidate'); - expect(response.header).to.have.property('etag'); + expect(response.header).to.have.property( + 'cache-control', + 'public, max-age=31536000, immutable' + ); + expect(response.header).not.to.have.property('etag'); }); }); diff --git a/test/server_integration/http/platform/cache.ts b/test/server_integration/http/platform/cache.ts index 6e1cd8ab39db01..80d0bbfebd5bd4 100644 --- a/test/server_integration/http/platform/cache.ts +++ b/test/server_integration/http/platform/cache.ts @@ -22,7 +22,7 @@ export default function ({ getService }: FtrProviderContext) { it('allows translation bundles to be cached', async () => { await supertest .get('/translations/en.json') - .expect('Cache-Control', 'must-revalidate') + .expect('Cache-Control', 'public, max-age=31536000, immutable') .expect(200); }); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/core/translations.ts b/x-pack/test_serverless/api_integration/test_suites/common/core/translations.ts index 26b4302bf2c714..b982799f4a71b2 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/core/translations.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/core/translations.ts @@ -17,8 +17,11 @@ export default function ({ getService }: FtrProviderContext) { expect(response.body.locale).to.eql('en'); expect(response.header).to.have.property('content-type', 'application/json; charset=utf-8'); - expect(response.header).to.have.property('cache-control', 'must-revalidate'); - expect(response.header).to.have.property('etag'); + expect(response.header).to.have.property( + 'cache-control', + 'public, max-age=31536000, immutable' + ); + expect(response.header).not.to.have.property('etag'); }); }); From 910e7f72523d0b015ec496aa967e76c61cf6c489 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:34:11 +0200 Subject: [PATCH 011/138] [Ent Search] Update Notebooks in console text (#181462) Closes https://github.com/elastic/search-docs-team/issues/100 - Update Introduction notebook to explain the what, how, and why of notebooks, clarify that available as preview (not actually runnable in our UI) currently, how to use in Colab and locally, link to Search Labs - Update notebook preview snippets to concise descriptions - Update link to search labs notebooks, update button text - General rewordings ## Before https://github.com/elastic/kibana/assets/32779855/a17d9b26-814b-4303-aac6-a8ef0a178ecf ## After https://github.com/elastic/kibana/assets/32779855/c8cfd685-c89b-4726-b89d-babcc7fbc3cf --- .../search_notebooks/common/constants.ts | 39 +++++++++++++++++-- .../components/search_labs_button_panel.tsx | 4 +- .../public/components/search_notebooks.tsx | 2 +- .../server/lib/notebook_catalog.ts | 12 +++--- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/search_notebooks/common/constants.ts b/x-pack/plugins/search_notebooks/common/constants.ts index 0a6a065751b82b..46cc4b339003b8 100644 --- a/x-pack/plugins/search_notebooks/common/constants.ts +++ b/x-pack/plugins/search_notebooks/common/constants.ts @@ -11,25 +11,56 @@ import { Notebook } from './types'; export const INTRODUCTION_NOTEBOOK: Notebook = { id: 'introduction', title: i18n.translate('xpack.searchNotebooks.introductionNotebook.title', { - defaultMessage: 'What are Jupyter Notebooks?', + defaultMessage: 'Jupyter notebooks', }), description: i18n.translate('xpack.searchNotebooks.introductionNotebook.description', { defaultMessage: - 'Jupyter Notebooks are an open-source document format for sharing interactive code embedded in narrative text.', + 'Learn all about Jupyter notebooks, how to preview them in the UI, and how to run them.', }), notebook: { cells: [ { cell_type: 'markdown', source: [ - '# What are Jupyter Notebooks\n', + '# What are Jupyter notebooks?\n', '\n', 'Jupyter Notebooks combine executable code and rich Markdown documentation in a single interactive document. Easy to run, edit and share, they enable collaboration in fields like data science, scientific computing, and machine learning.', + '\n', + '\n', + 'Notebooks are composed of cells, which can contain Markdown text like this, or Python code like the cell below.\n', ], }, { cell_type: 'code', - source: ['print("Hello world!!!")'], + source: ['print("Hello world!!!")\n'], + }, + { + cell_type: 'markdown', + source: [ + '\nNotebooks are a great way to test and prototype code, and share results with others. In our notebooks we use the official [Elasticsearch Python client](https://elasticsearch-py.readthedocs.io/en/latest/) to call the Elasticsearch APIs.', + ], + }, + { + cell_type: 'markdown', + source: [ + '## Elastic Jupyter notebooks\n', + '\n', + 'You can **preview** a number of our Jupyter notebooks right here in the UI. Check out the next section for how to **run** notebooks.\n', + '\nFind all of our available notebooks in the `elasticsearch-labs` [GitHub repository](https://github.com/elastic/elasticsearch-labs).', + '\n', + '## How to run notebooks\n', + '\n', + 'You can run notebooks in two ways:', + '\n', + '- **Run in Colab**: You can run all our notebooks in Google [Colab](https://colab.research.google.com), a free, zero configuration, in-browser notebook execution environment. Just click the `Open in Colab` button at the top of a notebook to test it in Colab.\n', + '- **Run locally**: You can also download the notebooks from the repository and run them locally using tools like [JupyterLab](https://jupyter.org/install).\n', + '\n', + 'ℹ️ Just make sure to copy your **Elasticsearch endpoint and API key** so the notebook can run against your deployment.\n', + '\n', + '## Learn more\n', + '\n', + 'Check out [Elastic Search Labs](https://www.elastic.co/search-labs) for all the latest advanced content for Elasticsearch users, including additional Python examples.', + ], }, ], }, diff --git a/x-pack/plugins/search_notebooks/public/components/search_labs_button_panel.tsx b/x-pack/plugins/search_notebooks/public/components/search_labs_button_panel.tsx index d0682aacd1b73b..53f982e9dc3711 100644 --- a/x-pack/plugins/search_notebooks/public/components/search_labs_button_panel.tsx +++ b/x-pack/plugins/search_notebooks/public/components/search_labs_button_panel.tsx @@ -13,7 +13,7 @@ export const SearchLabsButtonPanel = () => { { data-telemetry-id="console-notebooks-search-labs-btn" > {i18n.translate('xpack.searchNotebooks.searchLabsLink', { - defaultMessage: 'See more at Elastic Search Labs', + defaultMessage: 'Browse all our notebooks', })} diff --git a/x-pack/plugins/search_notebooks/public/components/search_notebooks.tsx b/x-pack/plugins/search_notebooks/public/components/search_notebooks.tsx index 312090de46755a..127de94877c22b 100644 --- a/x-pack/plugins/search_notebooks/public/components/search_notebooks.tsx +++ b/x-pack/plugins/search_notebooks/public/components/search_notebooks.tsx @@ -70,7 +70,7 @@ export const SearchNotebooks = () => { /> {i18n.translate('xpack.searchNotebooks.notebooksList.availableNotebooks.title', { - defaultMessage: 'Available Notebooks', + defaultMessage: 'Notebook previews', })} Date: Wed, 24 Apr 2024 13:43:58 +0200 Subject: [PATCH 012/138] [Cases] Fix failing test: `useCreateCaseModal ` (#181386) Fixes #174205 ## Summary The failing test was rendering the opened modal and not doing anything with it. I removed that block and the execution time locally went from 200+ ms to around 4. --- .../components/use_create_case_modal/index.test.tsx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/x-pack/plugins/cases/public/components/use_create_case_modal/index.test.tsx b/x-pack/plugins/cases/public/components/use_create_case_modal/index.test.tsx index fa7e0213166ee9..ab5bd1bd10dfee 100644 --- a/x-pack/plugins/cases/public/components/use_create_case_modal/index.test.tsx +++ b/x-pack/plugins/cases/public/components/use_create_case_modal/index.test.tsx @@ -7,7 +7,6 @@ import React from 'react'; import { renderHook, act } from '@testing-library/react-hooks'; -import { render, act as reactAct } from '@testing-library/react'; import { useKibana } from '../../common/lib/kibana'; import type { UseCreateCaseModalProps, UseCreateCaseModalReturnedValues } from '.'; @@ -19,8 +18,7 @@ jest.mock('../../common/lib/kibana'); const useKibanaMock = useKibana as jest.Mocked; const onCaseCreated = jest.fn(); -// FLAKY: https://github.com/elastic/kibana/issues/174205 -describe.skip('useCreateCaseModal', () => { +describe('useCreateCaseModal', () => { let navigateToApp: jest.Mock; beforeEach(() => { @@ -95,14 +93,6 @@ describe.skip('useCreateCaseModal', () => { act(() => { result.current.openModal(); - }); - - await reactAct(async () => { - const modal = result.current.modal; - render({modal}); - }); - - act(() => { result.current.modal.props.onSuccess({ id: 'case-id' }); }); From 2bd49f5ae4816c75a87a371e58ca2b7ea73d4bd4 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Wed, 24 Apr 2024 13:55:03 +0200 Subject: [PATCH 013/138] [ML] AIOps: Fix query string for the metric chart (#181314) ## Summary Fixes https://github.com/elastic/kibana/issues/179814 Query string from the Unified Search query bar was applying to the change point agg request, but wasn't in sync with the query service to correctly render the chart preview. With the PR the query string is synchronized and metric charts are rendered correctly. ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../change_point_detection_context.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx index eabfc277b7f9fa..75f4820316e309 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx @@ -132,7 +132,7 @@ export const ChangePointDetectionContextProvider: FC = ({ children }) => { const { uiSettings, data: { - query: { filterManager }, + query: { filterManager, queryString }, }, } = useAiopsAppContext(); @@ -241,11 +241,18 @@ export const ChangePointDetectionContextProvider: FC = ({ children }) => { if (requestParamsFromUrl.filters) { filterManager.setFilters(requestParamsFromUrl.filters); } + if (requestParamsFromUrl.query) { + queryString.setQuery(requestParamsFromUrl.query); + } if (globalFilters) { filterManager?.addFilters(globalFilters); } + return () => { + filterManager?.removeAll(); + queryString.clearQuery(); + }; }, - [requestParamsFromUrl.filters, filterManager] + [requestParamsFromUrl.filters, requestParamsFromUrl.query, filterManager, queryString] ); const combinedQuery = useMemo(() => { From 923b10baafd81f76321a7ae1f5105c183418fac9 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Wed, 24 Apr 2024 14:35:06 +0200 Subject: [PATCH 014/138] Fix `maps` plugin async route registration (#181518) ## Summary Async registration is bad, especially when promise rejections are not caught. The PR adapts the routes registration to be synchronous --- .../server/data_indexing/indexing_routes.ts | 5 ++-- x-pack/plugins/maps/server/mvt/mvt_routes.ts | 8 +++---- x-pack/plugins/maps/server/plugin.ts | 2 +- x-pack/plugins/maps/server/routes.ts | 24 ++++++++++++++----- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/maps/server/data_indexing/indexing_routes.ts b/x-pack/plugins/maps/server/data_indexing/indexing_routes.ts index 94fbb941655147..e7cb10427a2976 100644 --- a/x-pack/plugins/maps/server/data_indexing/indexing_routes.ts +++ b/x-pack/plugins/maps/server/data_indexing/indexing_routes.ts @@ -26,11 +26,11 @@ import { getMatchingIndexes } from './get_indexes_matching_pattern'; export function initIndexingRoutes({ router, logger, - dataPlugin, + getDataPlugin, }: { router: IRouter; logger: Logger; - dataPlugin: DataPluginStart; + getDataPlugin: () => Promise; securityPlugin?: SecurityPluginStart; }) { router.versioned @@ -58,6 +58,7 @@ export function initIndexingRoutes({ async (context, request, response) => { const coreContext = await context.core; const { index, mappings } = request.body; + const dataPlugin = await getDataPlugin(); const indexPatternsService = await dataPlugin.indexPatterns.dataViewsServiceFactory( coreContext.savedObjects.client, coreContext.elasticsearch.client.asCurrentUser, diff --git a/x-pack/plugins/maps/server/mvt/mvt_routes.ts b/x-pack/plugins/maps/server/mvt/mvt_routes.ts index 1cd53f02f83c6b..f768eb93dd9a36 100644 --- a/x-pack/plugins/maps/server/mvt/mvt_routes.ts +++ b/x-pack/plugins/maps/server/mvt/mvt_routes.ts @@ -27,11 +27,11 @@ const CACHE_TIMEOUT_SECONDS = 60 * 60; export function initMVTRoutes({ router, logger, - core, + getCore, }: { router: IRouter; logger: Logger; - core: CoreStart; + getCore: () => Promise; }) { router.versioned .get({ @@ -93,7 +93,7 @@ export function initMVTRoutes({ abortController: makeAbortController(request), body: tileRequest.body, context, - core, + core: await getCore(), executionContext: makeExecutionContext({ type: 'server', name: APP_ID, @@ -173,7 +173,7 @@ export function initMVTRoutes({ abortController: makeAbortController(request), body: tileRequest.body, context, - core, + core: await getCore(), executionContext: makeExecutionContext({ type: 'server', name: APP_ID, diff --git a/x-pack/plugins/maps/server/plugin.ts b/x-pack/plugins/maps/server/plugin.ts index dffd3e8a23aaaf..8d926952d1bd0e 100644 --- a/x-pack/plugins/maps/server/plugin.ts +++ b/x-pack/plugins/maps/server/plugin.ts @@ -145,7 +145,7 @@ export class MapsPlugin implements Plugin { ); } - setup(core: CoreSetup, plugins: SetupDeps) { + setup(core: CoreSetup, plugins: SetupDeps) { const getFilterMigrations = plugins.data.query.filterManager.getAllMigrations.bind( plugins.data.query.filterManager ); diff --git a/x-pack/plugins/maps/server/routes.ts b/x-pack/plugins/maps/server/routes.ts index 07bf8fefd9d142..32f7a9e6c18eaa 100644 --- a/x-pack/plugins/maps/server/routes.ts +++ b/x-pack/plugins/maps/server/routes.ts @@ -8,7 +8,7 @@ import { schema } from '@kbn/config-schema'; import fs from 'fs'; import path from 'path'; -import { CoreSetup, CoreStart, IRouter, Logger } from '@kbn/core/server'; +import { CoreSetup, IRouter, Logger } from '@kbn/core/server'; import { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import { INDEX_SETTINGS_API_PATH, FONTS_API_PATH } from '../common/constants'; import { getIndexPatternSettings } from './lib/get_index_pattern_settings'; @@ -16,10 +16,8 @@ import { initMVTRoutes } from './mvt/mvt_routes'; import { initIndexingRoutes } from './data_indexing/indexing_routes'; import { StartDeps } from './types'; -export async function initRoutes(coreSetup: CoreSetup, logger: Logger): Promise { +export function initRoutes(coreSetup: CoreSetup, logger: Logger) { const router: IRouter = coreSetup.http.createRouter(); - const [coreStart, { data: dataPlugin }]: [CoreStart, StartDeps] = - (await coreSetup.getStartServices()) as unknown as [CoreStart, StartDeps]; router.versioned .get({ @@ -109,6 +107,20 @@ export async function initRoutes(coreSetup: CoreSetup, logger: Logger): Promise< } ); - initMVTRoutes({ router, logger, core: coreStart }); - initIndexingRoutes({ router, logger, dataPlugin }); + initMVTRoutes({ + router, + logger, + getCore: async () => { + const [core] = await coreSetup.getStartServices(); + return core; + }, + }); + initIndexingRoutes({ + router, + logger, + getDataPlugin: async () => { + const [, { data }] = await coreSetup.getStartServices(); + return data; + }, + }); } From 58562c95656d37ad458d204f1e1810c63b7b2456 Mon Sep 17 00:00:00 2001 From: Pablo Machado Date: Wed, 24 Apr 2024 14:59:48 +0200 Subject: [PATCH 015/138] [SecuritySolutions] Fix entity analytics UI issues on dark mode (#181431) ## Summary We were using the wrong colour variables in a couple of places. These changes only impact the dark mode UI. Fixed pages: ![Screenshot 2024-04-23 at 14 55 20](https://github.com/elastic/kibana/assets/1490444/4b330c43-559d-4542-ab6b-46e98b69ce19) ![Screenshot 2024-04-23 at 14 36 12](https://github.com/elastic/kibana/assets/1490444/0b47b893-e9f2-4a54-b7c6-d055f2bb91a6) . --- .../components/file_picker_step.tsx | 2 +- .../shared/components/entity_table/columns.tsx | 6 +++--- .../components/side_panel/new_user_detail/columns.tsx | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality_file_uploader/components/file_picker_step.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality_file_uploader/components/file_picker_step.tsx index efe70c77f4c8f4..942b3a18c2c21b 100644 --- a/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality_file_uploader/components/file_picker_step.tsx +++ b/x-pack/plugins/security_solution/public/entity_analytics/components/asset_criticality_file_uploader/components/file_picker_step.tsx @@ -137,7 +137,7 @@ export const AssetCriticalityFilePickerStep: React.FC( {label ?? field} diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/new_user_detail/columns.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/new_user_detail/columns.tsx index da4e82976d5155..fc3aeb6aedd9f3 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/new_user_detail/columns.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/new_user_detail/columns.tsx @@ -7,7 +7,7 @@ import { css } from '@emotion/react'; import React from 'react'; -import { euiLightVars } from '@kbn/ui-theme'; +import { euiThemeVars } from '@kbn/ui-theme'; import type { EuiBasicTableColumn } from '@elastic/eui'; import { SourcererScopeName } from '../../../../common/store/sourcerer/model'; import { DefaultFieldRenderer } from '../../field_renderers/field_renderers'; @@ -21,8 +21,8 @@ const fieldColumn: EuiBasicTableColumn = { render: (label: string, { field }) => ( {label ?? field} From 707ec552d9a303dae2ebb4603e8fd4ca157ff6d4 Mon Sep 17 00:00:00 2001 From: Abdul Wahab Zahid Date: Wed, 24 Apr 2024 15:27:08 +0200 Subject: [PATCH 016/138] [Dataset quality] Pass breakdown field over to logs explorer from degraded docs chart (#181509) ## Summary The PR adds the `breakdownField` param in `LogsExplorerNavigationParams` so that when "Explorer data in Logs Explorer" is clicked on Degraded Docs chart on Dataset Quality flyout while the chart has a breakdown field selected, the field is passed over to Logs Explorer. https://github.com/elastic/kibana/assets/2748376/b380ac85-e40e-451b-983f-41c68f87ed7b --- .../observability/locators/logs_explorer.ts | 4 ++ .../common/locators/locators.test.ts | 48 +++++++++++++++++++ .../locators/utils/construct_locator_path.ts | 12 ++++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/packages/deeplinks/observability/locators/logs_explorer.ts b/packages/deeplinks/observability/locators/logs_explorer.ts index 1442e7afe02cda..39a28f91e7c110 100644 --- a/packages/deeplinks/observability/locators/logs_explorer.ts +++ b/packages/deeplinks/observability/locators/logs_explorer.ts @@ -68,6 +68,10 @@ export interface LogsExplorerNavigationParams extends SerializableRecord { * Optionally apply curated filter controls */ filterControls?: FilterControls; + /** + * Optionally set chart's breakdown field + */ + breakdownField?: string; } export interface LogsExplorerLocatorParams extends LogsExplorerNavigationParams { diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/locators.test.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/locators.test.ts index cdc50af3f54710..6761953c623258 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/locators.test.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/locators.test.ts @@ -99,6 +99,21 @@ describe('Observability Logs Explorer Locators', () => { }); }); + it('should allow specifying breakdown field', async () => { + const params: AllDatasetsLocatorParams = { + breakdownField: 'service.name', + }; + + const { allDatasetsLocator } = await setup(); + const location = await allDatasetsLocator.getLocation(params); + + expect(location).toMatchObject({ + app: OBSERVABILITY_LOGS_EXPLORER_APP_ID, + path: '/?pageState=(breakdownField:service.name,dataSourceSelection:(selectionType:all),v:2)', + state: {}, + }); + }); + it('should allow specifying columns', async () => { const params: AllDatasetsLocatorParams = { columns: [{ field: '_source', type: 'document-field' }], @@ -211,6 +226,22 @@ describe('Observability Logs Explorer Locators', () => { }); }); + it('should allow specifying breakdown field', async () => { + const params: ObsLogsExplorerDataViewLocatorParams = { + id: 'data-view-id', + breakdownField: 'service.name', + }; + + const { dataViewLocator } = await setup(); + const location = await dataViewLocator.getLocation(params); + + expect(location).toMatchObject({ + app: OBSERVABILITY_LOGS_EXPLORER_APP_ID, + path: `/?pageState=(breakdownField:service.name,dataSourceSelection:(selection:(dataView:(dataType:unresolved,id:data-view-id)),selectionType:dataView),v:2)`, + state: {}, + }); + }); + it('should allow specifying columns', async () => { const params: ObsLogsExplorerDataViewLocatorParams = { id: 'data-view-id', @@ -331,6 +362,23 @@ describe('Observability Logs Explorer Locators', () => { }); }); + it('should allow specifying breakdown field', async () => { + const params: SingleDatasetLocatorParams = { + integration, + dataset, + breakdownField: 'service.name', + }; + + const { singleDatasetLocator } = await setup(); + const location = await singleDatasetLocator.getLocation(params); + + expect(location).toMatchObject({ + app: OBSERVABILITY_LOGS_EXPLORER_APP_ID, + path: `/?pageState=(breakdownField:service.name,dataSourceSelection:(selection:(dataset:(name:'logs-test-*-*',title:test),name:Test),selectionType:unresolved),v:2)`, + state: {}, + }); + }); + it('should allow specifying columns', async () => { const params: SingleDatasetLocatorParams = { integration, diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/utils/construct_locator_path.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/utils/construct_locator_path.ts index eb96fb81ae2f3a..fd1a0249b32f87 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/utils/construct_locator_path.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/common/locators/utils/construct_locator_path.ts @@ -35,7 +35,16 @@ interface LocatorPathConstructionParams { export const constructLocatorPath = async (params: LocatorPathConstructionParams) => { const { dataSourceSelection, - locatorParams: { filterControls, filters, query, refreshInterval, timeRange, columns, origin }, + locatorParams: { + filterControls, + filters, + query, + refreshInterval, + timeRange, + columns, + origin, + breakdownField, + }, useHash, } = params; @@ -47,6 +56,7 @@ export const constructLocatorPath = async (params: LocatorPathConstructionParams query, refreshInterval, time: timeRange, + breakdownField, columns: columns?.map((column) => { return column.type === 'smart-field' ? SMART_FALLBACK_FIELDS[column.smartField] : column; }), From 42fa118b7dd4e0ec9108ca056d5f705d6f5617e9 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Wed, 24 Apr 2024 16:38:24 +0300 Subject: [PATCH 017/138] [Cases] Populate user info from fake requests (#180671) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Set a system user name when the request is fake. This is helpful to show user information to the case created by the case action. ## Testing 1. Create a rule with a case action 2. In the created case verify that the `elastic/kibana` user is shown and not the `Unknown` user Screenshot 2024-04-22 at 10 37 46 PM ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [x] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../actions_client/actions_client.mock.ts | 2 +- .../actions_authorization.mock.ts | 2 +- x-pack/plugins/actions/server/feature.ts | 2 +- x-pack/plugins/actions/tsconfig.json | 3 +- .../plugins/cases/common/constants/index.ts | 1 + .../cases/server/client/factory.test.ts | 124 ++++++++++++++++++ x-pack/plugins/cases/server/client/factory.ts | 10 +- x-pack/plugins/cases/server/client/mocks.ts | 36 ++++- x-pack/plugins/cases/server/plugin.test.ts | 4 +- x-pack/plugins/cases/tsconfig.json | 1 + 10 files changed, 176 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/cases/server/client/factory.test.ts diff --git a/x-pack/plugins/actions/server/actions_client/actions_client.mock.ts b/x-pack/plugins/actions/server/actions_client/actions_client.mock.ts index d58476738b9be5..433e3850cae2a6 100644 --- a/x-pack/plugins/actions/server/actions_client/actions_client.mock.ts +++ b/x-pack/plugins/actions/server/actions_client/actions_client.mock.ts @@ -6,7 +6,7 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import { ActionsClient } from './actions_client'; +import type { ActionsClient } from './actions_client'; type ActionsClientContract = PublicMethodsOf; export type ActionsClientMock = jest.Mocked; diff --git a/x-pack/plugins/actions/server/authorization/actions_authorization.mock.ts b/x-pack/plugins/actions/server/authorization/actions_authorization.mock.ts index 20e95f1560c5a8..ba106d014c28d7 100644 --- a/x-pack/plugins/actions/server/authorization/actions_authorization.mock.ts +++ b/x-pack/plugins/actions/server/authorization/actions_authorization.mock.ts @@ -6,7 +6,7 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import { ActionsAuthorization } from './actions_authorization'; +import type { ActionsAuthorization } from './actions_authorization'; export type ActionsAuthorizationMock = jest.Mocked>; diff --git a/x-pack/plugins/actions/server/feature.ts b/x-pack/plugins/actions/server/feature.ts index b44aaf61cad629..9fc48b705d25bf 100644 --- a/x-pack/plugins/actions/server/feature.ts +++ b/x-pack/plugins/actions/server/feature.ts @@ -5,8 +5,8 @@ * 2.0. */ +import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common'; import { i18n } from '@kbn/i18n'; -import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server'; import { ACTION_SAVED_OBJECT_TYPE, ACTION_TASK_PARAMS_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/actions/tsconfig.json b/x-pack/plugins/actions/tsconfig.json index 76ca916bc0ad03..d060287d241430 100644 --- a/x-pack/plugins/actions/tsconfig.json +++ b/x-pack/plugins/actions/tsconfig.json @@ -46,7 +46,8 @@ "@kbn/actions-types", "@kbn/core-http-server", "@kbn/core-test-helpers-kbn-server", - "@kbn/security-plugin-types-server" + "@kbn/security-plugin-types-server", + "@kbn/core-application-common" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/cases/common/constants/index.ts b/x-pack/plugins/cases/common/constants/index.ts index 6b6b914826b61c..a8868010d23120 100644 --- a/x-pack/plugins/cases/common/constants/index.ts +++ b/x-pack/plugins/cases/common/constants/index.ts @@ -192,6 +192,7 @@ export const GET_CONNECTORS_CONFIGURE_API_TAG = 'casesGetConnectorsConfigure'; export const DEFAULT_USER_SIZE = 10; export const MAX_ASSIGNEES_PER_CASE = 10; export const NO_ASSIGNEES_FILTERING_KEYWORD = 'none'; +export const KIBANA_SYSTEM_USERNAME = 'elastic/kibana'; /** * Delays diff --git a/x-pack/plugins/cases/server/client/factory.test.ts b/x-pack/plugins/cases/server/client/factory.test.ts new file mode 100644 index 00000000000000..69147e888aeec4 --- /dev/null +++ b/x-pack/plugins/cases/server/client/factory.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock, httpServerMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { CasesClientFactory } from './factory'; +import { createCasesClientFactoryMockArgs } from './mocks'; +import { createCasesClient } from './client'; +import type { FakeRawRequest } from '@kbn/core-http-server'; +import { CoreKibanaRequest } from '@kbn/core-http-router-server-internal'; + +jest.mock('./client'); + +describe('CasesClientFactory', () => { + const coreStart = coreMock.createStart(); + const request = httpServerMock.createKibanaRequest(); + + const rawRequest: FakeRawRequest = { + headers: {}, + path: '/', + }; + + const fakeRequest = CoreKibanaRequest.from(rawRequest); + const createCasesClientMocked = createCasesClient as jest.Mock; + const logger = loggingSystemMock.createLogger(); + const args = createCasesClientFactoryMockArgs(); + let casesClientFactory: CasesClientFactory; + + args.featuresPluginStart.getKibanaFeatures.mockReturnValue([]); + + beforeEach(() => { + casesClientFactory = new CasesClientFactory(logger); + casesClientFactory.initialize(args); + jest.clearAllMocks(); + }); + + describe('user info', () => { + it('constructs the user info from user profiles', async () => { + const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + args.securityPluginStart.userProfiles.getCurrent.mockResolvedValueOnce({ + // @ts-expect-error: not all fields are needed + user: { username: 'my_user', full_name: 'My user', email: 'elastic@elastic.co' }, + }); + + await casesClientFactory.create({ + request, + savedObjectsService: coreStart.savedObjects, + scopedClusterClient, + }); + + expect(args.securityPluginStart.userProfiles.getCurrent).toHaveBeenCalled(); + expect(args.securityPluginStart.authc.getCurrentUser).not.toHaveBeenCalled(); + expect(createCasesClientMocked.mock.calls[0][0].user).toEqual({ + username: 'my_user', + full_name: 'My user', + email: 'elastic@elastic.co', + }); + }); + + it('constructs the user info from the authc service if the user profile is not available', async () => { + const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + // @ts-expect-error: not all fields are needed + args.securityPluginStart.authc.getCurrentUser.mockReturnValueOnce({ + username: 'my_user_2', + full_name: 'My user 2', + email: 'elastic2@elastic.co', + }); + + await casesClientFactory.create({ + request, + savedObjectsService: coreStart.savedObjects, + scopedClusterClient, + }); + + expect(args.securityPluginStart.userProfiles.getCurrent).toHaveBeenCalled(); + expect(args.securityPluginStart.authc.getCurrentUser).toHaveBeenCalled(); + expect(createCasesClientMocked.mock.calls[0][0].user).toEqual({ + username: 'my_user_2', + full_name: 'My user 2', + email: 'elastic2@elastic.co', + }); + }); + + it('constructs the user info from fake requests correctly', async () => { + const scopedClusterClient = + coreStart.elasticsearch.client.asScoped(fakeRequest).asCurrentUser; + + await casesClientFactory.create({ + request: fakeRequest, + savedObjectsService: coreStart.savedObjects, + scopedClusterClient, + }); + + expect(args.securityPluginStart.userProfiles.getCurrent).toHaveBeenCalled(); + expect(args.securityPluginStart.authc.getCurrentUser).toHaveBeenCalled(); + expect(createCasesClientMocked.mock.calls[0][0].user).toEqual({ + username: 'elastic/kibana', + full_name: null, + email: null, + }); + }); + + it('return null for all user fields if it cannot find the user info', async () => { + const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + + await casesClientFactory.create({ + request, + savedObjectsService: coreStart.savedObjects, + scopedClusterClient, + }); + + expect(args.securityPluginStart.userProfiles.getCurrent).toHaveBeenCalled(); + expect(args.securityPluginStart.authc.getCurrentUser).toHaveBeenCalled(); + expect(createCasesClientMocked.mock.calls[0][0].user).toEqual({ + username: null, + full_name: null, + email: null, + }); + }); + }); +}); diff --git a/x-pack/plugins/cases/server/client/factory.ts b/x-pack/plugins/cases/server/client/factory.ts index 137cf69e0763a2..5bb04c1da9e867 100644 --- a/x-pack/plugins/cases/server/client/factory.ts +++ b/x-pack/plugins/cases/server/client/factory.ts @@ -34,7 +34,7 @@ import type { import type { PublicMethodsOf } from '@kbn/utility-types'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; import type { FilesStart } from '@kbn/files-plugin/server'; -import { SAVED_OBJECT_TYPES } from '../../common/constants'; +import { KIBANA_SYSTEM_USERNAME, SAVED_OBJECT_TYPES } from '../../common/constants'; import { Authorization } from '../authorization/authorization'; import { CaseConfigureService, @@ -286,6 +286,14 @@ export class CasesClientFactory { this.logger.debug(`Failed to retrieve user info from authc: ${error}`); } + if (request.isFakeRequest) { + return { + username: KIBANA_SYSTEM_USERNAME, + full_name: null, + email: null, + }; + } + return { username: null, full_name: null, diff --git a/x-pack/plugins/cases/server/client/mocks.ts b/x-pack/plugins/cases/server/client/mocks.ts index 608a9ae2ff5100..3de350f5a39817 100644 --- a/x-pack/plugins/cases/server/client/mocks.ts +++ b/x-pack/plugins/cases/server/client/mocks.ts @@ -9,12 +9,20 @@ import type { PublicContract, PublicMethodsOf } from '@kbn/utility-types'; import { loggingSystemMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; import type { ISavedObjectsSerializer } from '@kbn/core-saved-objects-server'; -import { createFileServiceMock } from '@kbn/files-plugin/server/mocks'; +import { + createFileServiceFactoryMock, + createFileServiceMock, +} from '@kbn/files-plugin/server/mocks'; import { securityMock } from '@kbn/security-plugin/server/mocks'; import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; import { makeLensEmbeddableFactory } from '@kbn/lens-plugin/server/embeddable/make_lens_embeddable_factory'; import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; - +import { spacesMock } from '@kbn/spaces-plugin/server/mocks'; +import { featuresPluginMock } from '@kbn/features-plugin/server/mocks'; +import { actionsMock } from '@kbn/actions-plugin/server/mocks'; +import { notificationsMock } from '@kbn/notifications-plugin/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import type { CasesSearchRequest } from '../../common/types/api'; import type { CasesClient, CasesClientInternal } from '.'; import type { AttachmentsSubClient } from './attachments/client'; @@ -214,6 +222,30 @@ export const createCasesClientMockArgs = () => { }; }; +export const createCasesClientFactoryMockArgs = () => { + return { + securityPluginSetup: securityMock.createSetup(), + securityPluginStart: securityMock.createStart(), + spacesPluginStart: spacesMock.createStart(), + featuresPluginStart: featuresPluginMock.createSetup(), + actionsPluginStart: actionsMock.createStart(), + licensingPluginStart: licensingMock.createStart(), + notifications: notificationsMock.createStart(), + ruleRegistry: { getRacClientWithRequest: jest.fn(), alerting: alertsMock.createStart() }, + filesPluginStart: { fileServiceFactory: createFileServiceFactoryMock() }, + publicBaseUrl: 'https//example.com', + lensEmbeddableFactory: jest.fn().mockReturnValue( + makeLensEmbeddableFactory( + () => ({}), + () => ({}), + {} + ) + ), + externalReferenceAttachmentTypeRegistry: createExternalReferenceAttachmentTypeRegistryMock(), + persistableStateAttachmentTypeRegistry: createPersistableStateAttachmentTypeRegistryMock(), + }; +}; + export const createCasesClientMockSearchRequest = ( overwrites?: CasesSearchRequest ): CasesSearchRequest => ({ diff --git a/x-pack/plugins/cases/server/plugin.test.ts b/x-pack/plugins/cases/server/plugin.test.ts index 8c669f6de0e68c..ac328f25de391f 100644 --- a/x-pack/plugins/cases/server/plugin.test.ts +++ b/x-pack/plugins/cases/server/plugin.test.ts @@ -11,7 +11,7 @@ import { coreMock } from '@kbn/core/server/mocks'; import { usageCollectionPluginMock } from '@kbn/usage-collection-plugin/server/mocks'; import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { featuresPluginMock } from '@kbn/features-plugin/server/mocks'; -import { createFilesSetupMock } from '@kbn/files-plugin/server/mocks'; +import { createFileServiceFactoryMock, createFilesSetupMock } from '@kbn/files-plugin/server/mocks'; import { securityMock } from '@kbn/security-plugin/server/mocks'; import { makeLensEmbeddableFactory } from '@kbn/lens-plugin/server/embeddable/make_lens_embeddable_factory'; import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; @@ -69,7 +69,7 @@ describe('Cases Plugin', () => { pluginsStart = { licensing: licensingMock.createStart(), actions: actionsMock.createStart(), - files: { fileServiceFactory: { asScoped: jest.fn(), asInternal: jest.fn() } }, + files: { fileServiceFactory: createFileServiceFactoryMock() }, features: featuresPluginMock.createStart(), security: securityMock.createStart(), notifications: notificationsMock.createStart(), diff --git a/x-pack/plugins/cases/tsconfig.json b/x-pack/plugins/cases/tsconfig.json index b5a426c3f9c0bc..535f4e5e106dc2 100644 --- a/x-pack/plugins/cases/tsconfig.json +++ b/x-pack/plugins/cases/tsconfig.json @@ -74,6 +74,7 @@ "@kbn/core-logging-server-mocks", "@kbn/core-logging-browser-mocks", "@kbn/data-views-plugin", + "@kbn/core-http-router-server-internal", ], "exclude": [ "target/**/*", From 05478009c69f5993207f8db1f93f1230f4006a96 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Wed, 24 Apr 2024 16:09:46 +0200 Subject: [PATCH 018/138] [HTTP/OAS] Support `deprecated` field (#181240) --- packages/kbn-config-schema/index.ts | 2 ++ .../kbn-config-schema/src/oas_meta_fields.ts | 1 + .../kbn-config-schema/src/types/type.test.ts | 24 ++++++++++++++----- packages/kbn-config-schema/src/types/type.ts | 14 +++++++++-- .../__snapshots__/generate_oas.test.ts.snap | 4 ++++ .../src/generate_oas.test.util.ts | 7 +++++- .../post_process_mutations/mutations/index.ts | 3 ++- .../post_process_mutations/mutations/utils.ts | 8 +++++++ 8 files changed, 53 insertions(+), 10 deletions(-) diff --git a/packages/kbn-config-schema/index.ts b/packages/kbn-config-schema/index.ts index dfb701f0a039cc..c5b46932463dbe 100644 --- a/packages/kbn-config-schema/index.ts +++ b/packages/kbn-config-schema/index.ts @@ -246,6 +246,7 @@ export type Schema = typeof schema; import { META_FIELD_X_OAS_REF_ID, META_FIELD_X_OAS_OPTIONAL, + META_FIELD_X_OAS_DEPRECATED, META_FIELD_X_OAS_MAX_LENGTH, META_FIELD_X_OAS_MIN_LENGTH, META_FIELD_X_OAS_GET_ADDITIONAL_PROPERTIES, @@ -254,6 +255,7 @@ import { export const metaFields = Object.freeze({ META_FIELD_X_OAS_REF_ID, META_FIELD_X_OAS_OPTIONAL, + META_FIELD_X_OAS_DEPRECATED, META_FIELD_X_OAS_MAX_LENGTH, META_FIELD_X_OAS_MIN_LENGTH, META_FIELD_X_OAS_GET_ADDITIONAL_PROPERTIES, diff --git a/packages/kbn-config-schema/src/oas_meta_fields.ts b/packages/kbn-config-schema/src/oas_meta_fields.ts index 44422793558d4c..ad04f9b30ef16b 100644 --- a/packages/kbn-config-schema/src/oas_meta_fields.ts +++ b/packages/kbn-config-schema/src/oas_meta_fields.ts @@ -16,3 +16,4 @@ export const META_FIELD_X_OAS_MAX_LENGTH = 'x-oas-max-length' as const; export const META_FIELD_X_OAS_GET_ADDITIONAL_PROPERTIES = 'x-oas-get-additional-properties' as const; export const META_FIELD_X_OAS_REF_ID = 'x-oas-ref-id' as const; +export const META_FIELD_X_OAS_DEPRECATED = 'x-oas-deprecated' as const; diff --git a/packages/kbn-config-schema/src/types/type.test.ts b/packages/kbn-config-schema/src/types/type.test.ts index 4d6636a55b9ca9..a4784af2e8b62f 100644 --- a/packages/kbn-config-schema/src/types/type.test.ts +++ b/packages/kbn-config-schema/src/types/type.test.ts @@ -9,7 +9,7 @@ import { get } from 'lodash'; import { internals } from '../internals'; import { Type, TypeOptions } from './type'; -import { META_FIELD_X_OAS_REF_ID } from '../oas_meta_fields'; +import { META_FIELD_X_OAS_REF_ID, META_FIELD_X_OAS_DEPRECATED } from '../oas_meta_fields'; class MyType extends Type { constructor(opts: TypeOptions = {}) { @@ -17,9 +17,21 @@ class MyType extends Type { } } -test('meta', () => { - const type = new MyType({ meta: { description: 'my description', id: 'foo' } }); - const meta = type.getSchema().describe(); - expect(get(meta, 'flags.description')).toBe('my description'); - expect(get(meta, `metas[0].${META_FIELD_X_OAS_REF_ID}`)).toBe('foo'); +describe('meta', () => { + it('sets meta when provided', () => { + const type = new MyType({ + meta: { description: 'my description', id: 'foo', deprecated: true }, + }); + const meta = type.getSchema().describe(); + expect(get(meta, 'flags.description')).toBe('my description'); + expect(get(meta, `metas[0].${META_FIELD_X_OAS_REF_ID}`)).toBe('foo'); + expect(get(meta, `metas[1].${META_FIELD_X_OAS_DEPRECATED}`)).toBe(true); + }); + + it('does not set meta when no provided', () => { + const type = new MyType(); + const meta = type.getSchema().describe(); + expect(get(meta, 'flags.description')).toBeUndefined(); + expect(get(meta, 'metas')).toBeUndefined(); + }); }); diff --git a/packages/kbn-config-schema/src/types/type.ts b/packages/kbn-config-schema/src/types/type.ts index 1e312cc7adc7e9..80ed3f90fdd2a8 100644 --- a/packages/kbn-config-schema/src/types/type.ts +++ b/packages/kbn-config-schema/src/types/type.ts @@ -7,16 +7,23 @@ */ import type { AnySchema, CustomValidator, ErrorReport } from 'joi'; -import { META_FIELD_X_OAS_REF_ID } from '../oas_meta_fields'; +import { META_FIELD_X_OAS_DEPRECATED, META_FIELD_X_OAS_REF_ID } from '../oas_meta_fields'; import { SchemaTypeError, ValidationError } from '../errors'; import { Reference } from '../references'; -/** Meta fields used when introspecting runtime validation */ +/** + * Meta fields used when introspecting runtime validation. Most notably for + * generating OpenAPI spec. + */ export interface TypeMeta { /** * A human-friendly description of this type to be used in documentation. */ description?: string; + /** + * Whether this field is deprecated. + */ + deprecated?: boolean; /** * A string that uniquely identifies this schema. Used when generating OAS * to create refs instead of inline schemas. @@ -108,6 +115,9 @@ export abstract class Type { if (options.meta.id) { schema = schema.meta({ [META_FIELD_X_OAS_REF_ID]: options.meta.id }); } + if (options.meta.deprecated) { + schema = schema.meta({ [META_FIELD_X_OAS_DEPRECATED]: true }); + } } // Attach generic error handler only if it hasn't been attached yet since diff --git a/packages/kbn-router-to-openapispec/src/__snapshots__/generate_oas.test.ts.snap b/packages/kbn-router-to-openapispec/src/__snapshots__/generate_oas.test.ts.snap index 42deee2907f43c..aac31188c49855 100644 --- a/packages/kbn-router-to-openapispec/src/__snapshots__/generate_oas.test.ts.snap +++ b/packages/kbn-router-to-openapispec/src/__snapshots__/generate_oas.test.ts.snap @@ -155,6 +155,10 @@ Object { "schema": Object { "additionalProperties": false, "properties": Object { + "deprecatedFoo": Object { + "deprecated": true, + "type": "string", + }, "foo": Object { "type": "string", }, diff --git a/packages/kbn-router-to-openapispec/src/generate_oas.test.util.ts b/packages/kbn-router-to-openapispec/src/generate_oas.test.util.ts index e435065f0089f5..41a2458619000e 100644 --- a/packages/kbn-router-to-openapispec/src/generate_oas.test.util.ts +++ b/packages/kbn-router-to-openapispec/src/generate_oas.test.util.ts @@ -79,7 +79,12 @@ const getVersionedRouterDefaults = () => ({ fn: jest.fn(), options: { validate: { - request: { body: schema.object({ foo: schema.string() }) }, + request: { + body: schema.object({ + foo: schema.string(), + deprecatedFoo: schema.maybe(schema.string({ meta: { deprecated: true } })), + }), + }, response: { [200]: { body: schema.object({ fooResponse: schema.string() }) }, }, diff --git a/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/index.ts b/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/index.ts index 4f5b31030695e9..e200ba7d1ec278 100644 --- a/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/index.ts +++ b/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/index.ts @@ -10,7 +10,7 @@ import Joi from 'joi'; import { metaFields } from '@kbn/config-schema'; import type { OpenAPIV3 } from 'openapi-types'; import { parse } from '../../parse'; -import { deleteField, stripBadDefault } from './utils'; +import { deleteField, stripBadDefault, processDeprecated } from './utils'; import { IContext } from '../context'; const { @@ -62,6 +62,7 @@ export const processMap = (ctx: IContext, schema: OpenAPIV3.SchemaObject): void }; export const processAny = (schema: OpenAPIV3.SchemaObject): void => { + processDeprecated(schema); stripBadDefault(schema); }; diff --git a/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/utils.ts b/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/utils.ts index 5bad28b276d103..eacc005936a28b 100644 --- a/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/utils.ts +++ b/packages/kbn-router-to-openapispec/src/oas_converter/kbn_config_schema/post_process_mutations/mutations/utils.ts @@ -7,6 +7,7 @@ */ import type { OpenAPIV3 } from 'openapi-types'; +import { metaFields } from '@kbn/config-schema'; export const stripBadDefault = (schema: OpenAPIV3.SchemaObject): void => { if (schema.default?.special === 'deep') { @@ -26,6 +27,13 @@ export const stripBadDefault = (schema: OpenAPIV3.SchemaObject): void => { } }; +export const processDeprecated = (schema: OpenAPIV3.SchemaObject): void => { + if (metaFields.META_FIELD_X_OAS_DEPRECATED in schema) { + schema.deprecated = true; + deleteField(schema, metaFields.META_FIELD_X_OAS_DEPRECATED); + } +}; + /** Just for type convenience */ export const deleteField = (schema: Record, field: string): void => { delete schema[field]; From d5999c339a615a4e7cbd23741747dffa984da678 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Wed, 24 Apr 2024 16:29:06 +0200 Subject: [PATCH 019/138] [ML] Fix responsive layout for Trained Models table (#181541) ## Summary Closes https://github.com/elastic/kibana/issues/181530 - Sets percentage width for all columns - Sets responsive breakpoint - Makes Deployment stats table responsive as well ![Apr-24-2024 12-16-48](https://github.com/elastic/kibana/assets/5236598/2a14ffb9-de15-45e9-b8bc-276e10080864) ### Checklist - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../nodes_overview/allocated_models.tsx | 29 ++++++++++--------- .../model_management/model_actions.tsx | 6 ++-- .../model_management/models_list.tsx | 15 +++++----- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/allocated_models.tsx b/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/allocated_models.tsx index f0ff64eae64457..4a31112255d925 100644 --- a/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/allocated_models.tsx +++ b/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/allocated_models.tsx @@ -41,21 +41,22 @@ export const AllocatedModels: FC = ({ const columns: Array> = [ { + width: '10%', id: 'deployment_id', field: 'deployment_id', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.deploymentIdHeader', { defaultMessage: 'ID', }), - width: '150px', sortable: true, truncateText: false, + isExpander: false, 'data-test-subj': 'mlAllocatedModelsTableDeploymentId', }, { + width: '8%', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelRoutingStateHeader', { defaultMessage: 'Routing state', }), - width: '100px', 'data-test-subj': 'mlAllocatedModelsTableRoutingState', render: (v: AllocatedModel) => { const { routing_state: routingState, reason } = v.node.routing_state; @@ -68,32 +69,32 @@ export const AllocatedModels: FC = ({ }, }, { + width: '8%', id: 'node_name', field: 'node.name', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.nodeNameHeader', { defaultMessage: 'Node name', }), - width: '150px', sortable: true, truncateText: false, 'data-test-subj': 'mlAllocatedModelsTableNodeName', }, { + width: '10%', id: 'model_id', field: 'model_id', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelNameHeader', { defaultMessage: 'Name', }), - width: '250px', sortable: true, truncateText: false, 'data-test-subj': 'mlAllocatedModelsTableName', }, { + width: '8%', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelSizeHeader', { defaultMessage: 'Size', }), - width: '100px', truncateText: true, 'data-test-subj': 'mlAllocatedModelsTableSize', render: (v: AllocatedModel) => { @@ -101,6 +102,7 @@ export const AllocatedModels: FC = ({ }, }, { + width: '8%', name: ( = ({ ), - width: '100px', truncateText: false, 'data-test-subj': 'mlAllocatedModelsTableAllocation', render: (v: AllocatedModel) => { @@ -129,6 +130,7 @@ export const AllocatedModels: FC = ({ }, }, { + width: '8%', name: ( = ({ ), field: 'node.throughput_last_minute', - width: '100px', truncateText: false, 'data-test-subj': 'mlAllocatedModelsTableThroughput', }, { + width: '8%', name: ( = ({ ), - width: '100px', truncateText: false, 'data-test-subj': 'mlAllocatedModelsTableAvgInferenceTime', render: (v: AllocatedModel) => { @@ -196,56 +197,56 @@ export const AllocatedModels: FC = ({ }, }, { + width: '8%', name: i18n.translate( 'xpack.ml.trainedModels.nodesList.modelsList.modelInferenceCountHeader', { defaultMessage: 'Inference count', } ), - width: '100px', 'data-test-subj': 'mlAllocatedModelsTableInferenceCount', render: (v: AllocatedModel) => { return v.node.inference_count; }, }, { + width: '12%', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelStartTimeHeader', { defaultMessage: 'Start time', }), - width: '200px', 'data-test-subj': 'mlAllocatedModelsTableStartedTime', render: (v: AllocatedModel) => { return dateFormatter(v.node.start_time); }, }, { + width: '12%', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.modelLastAccessHeader', { defaultMessage: 'Last access', }), - width: '200px', 'data-test-subj': 'mlAllocatedModelsTableInferenceCount', render: (v: AllocatedModel) => { return v.node.last_access ? dateFormatter(v.node.last_access) : '-'; }, }, { + width: '8%', name: i18n.translate( 'xpack.ml.trainedModels.nodesList.modelsList.modelNumberOfPendingRequestsHeader', { defaultMessage: 'Pending requests', } ), - width: '100px', 'data-test-subj': 'mlAllocatedModelsTableNumberOfPendingRequests', render: (v: AllocatedModel) => { return v.node.number_of_pending_requests; }, }, { + width: '8%', name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsList.errorCountHeader', { defaultMessage: 'Errors', }), - width: '60px', 'data-test-subj': 'mlAllocatedModelsTableErrorCount', render: (v: AllocatedModel) => { return v.node.error_count ?? 0; @@ -255,6 +256,7 @@ export const AllocatedModels: FC = ({ return ( + responsiveBreakpoint={'xl'} allowNeutralSort={false} columns={columns} items={models} @@ -264,7 +266,6 @@ export const AllocatedModels: FC = ({ })} onTableChange={() => {}} data-test-subj={'mlNodesAllocatedModels'} - css={{ overflow: 'auto' }} /> ); }; diff --git a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx index c3b726d8bd2d6e..5d690a7a58fd89 100644 --- a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx +++ b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx @@ -199,9 +199,8 @@ export function useModelActions({ } ), 'data-test-subj': 'mlModelsTableRowStartDeploymentAction', - // @ts-ignore EUI has a type check issue when type "button" is combined with an icon. icon: 'play', - type: 'button', + type: 'icon', isPrimary: true, enabled: (item) => { return canStartStopTrainedModels && !isLoading && item.state !== MODEL_STATE.DOWNLOADING; @@ -409,9 +408,8 @@ export function useModelActions({ defaultMessage: 'Download', }), 'data-test-subj': 'mlModelsTableRowDownloadModelAction', - // @ts-ignore EUI has a type check issue when type "button" is combined with an icon. icon: 'download', - type: 'button', + type: 'icon', isPrimary: true, available: (item) => canCreateTrainedModels && diff --git a/x-pack/plugins/ml/public/application/model_management/models_list.tsx b/x-pack/plugins/ml/public/application/model_management/models_list.tsx index 8f4a9ae1e55f14..242db5904252da 100644 --- a/x-pack/plugins/ml/public/application/model_management/models_list.tsx +++ b/x-pack/plugins/ml/public/application/model_management/models_list.tsx @@ -486,7 +486,7 @@ export const ModelsList: FC = ({ const columns: Array> = [ { align: 'left', - width: '40px', + width: '32px', isExpander: true, render: (item: ModelItem) => { if (!item.stats) { @@ -512,7 +512,7 @@ export const ModelsList: FC = ({ }, { name: modelIdColumnName, - width: '215px', + width: '15%', sortable: ({ model_id: modelId }: ModelItem) => modelId, truncateText: false, textOnly: false, @@ -533,7 +533,6 @@ export const ModelsList: FC = ({ }, }, { - width: '300px', name: i18n.translate('xpack.ml.trainedModels.modelsList.modelDescriptionHeader', { defaultMessage: 'Description', }), @@ -567,6 +566,7 @@ export const ModelsList: FC = ({ }, }, { + width: '15%', field: ModelsTableToConfigMapping.type, name: i18n.translate('xpack.ml.trainedModels.modelsList.typeHeader', { defaultMessage: 'Type', @@ -586,9 +586,9 @@ export const ModelsList: FC = ({ ), 'data-test-subj': 'mlModelsTableColumnType', - width: '130px', }, { + width: '10%', field: 'state', name: i18n.translate('xpack.ml.trainedModels.modelsList.stateHeader', { defaultMessage: 'State', @@ -604,9 +604,9 @@ export const ModelsList: FC = ({ ) : null; }, 'data-test-subj': 'mlModelsTableColumnDeploymentState', - width: '130px', }, { + width: '20%', field: ModelsTableToConfigMapping.createdAt, name: i18n.translate('xpack.ml.trainedModels.modelsList.createdAtHeader', { defaultMessage: 'Created at', @@ -615,10 +615,9 @@ export const ModelsList: FC = ({ render: (v: number) => dateFormatter(v), sortable: true, 'data-test-subj': 'mlModelsTableColumnCreatedAt', - width: '210px', }, { - width: '150px', + width: '15%', name: i18n.translate('xpack.ml.trainedModels.modelsList.actionsHeader', { defaultMessage: 'Actions', }), @@ -768,7 +767,7 @@ export const ModelsList: FC = ({
- css={{ overflowX: 'auto' }} + responsiveBreakpoint={'xl'} allowNeutralSort={false} columns={columns} itemIdToExpandedRowMap={itemIdToExpandedRowMap} From d635c6deeba5891b548eea9e04fb7e144cc9eac5 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:14:49 +0200 Subject: [PATCH 020/138] [Fleet] allow fleet-server agent upgrade to newer than fleet-server (#181575) ## Summary Closes https://github.com/elastic/kibana/issues/181394 Allow upgrade fleet-server even if fleet-server version is older than the upgrade version. See testing steps in the linked issue. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../agent_upgrade_modal/index.test.tsx | 37 +++++++++++++++++++ .../components/agent_upgrade_modal/index.tsx | 5 +++ 2 files changed, 42 insertions(+) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.test.tsx index dc08d052a91522..a71329168d0546 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.test.tsx @@ -334,6 +334,43 @@ describe('AgentUpgradeAgentModal', () => { expect(el).not.toBeDisabled(); }); }); + + it('should enable submit button for a single fleet-server when version is greater than maxFleetServerVersion', async () => { + mockSendGetAgentsAvailableVersions.mockClear(); + mockSendGetAgentsAvailableVersions.mockResolvedValue({ + data: { + items: ['8.10.4', '8.10.2', '8.9.0', '8.8.0'], + }, + }); + mockSendAllFleetServerAgents.mockResolvedValue({ + allFleetServerAgents: [ + { id: 'fleet-server', local_metadata: { elastic: { agent: { version: '8.9.0' } } } }, + ] as any, + }); + + const { utils } = renderAgentUpgradeAgentModal({ + agents: [ + { + id: 'fleet-server', + local_metadata: { + elastic: { + agent: { version: '8.9.0', upgradeable: true }, + }, + host: { hostname: 'host00001' }, + }, + }, + ] as any, + agentCount: 1, + }); + + await waitFor(() => { + const container = utils.getByTestId('agentUpgradeModal.VersionCombobox'); + const input = within(container).getByRole('combobox'); + expect(input?.value).toEqual('8.10.2'); + const el = utils.getByTestId('confirmModalConfirmButton'); + expect(el).toBeEnabled(); + }); + }); }); describe('restart upgrade', () => { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx index 89531c2b011815..a82cef810f9cc4 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/components/agent_upgrade_modal/index.tsx @@ -287,6 +287,9 @@ export const AgentUpgradeAgentModal: React.FunctionComponent agent.id).includes(agents[0].id); + const isSubmitButtonDisabled = useMemo( () => isSubmitting || @@ -294,6 +297,7 @@ export const AgentUpgradeAgentModal: React.FunctionComponent Date: Wed, 24 Apr 2024 17:21:25 +0200 Subject: [PATCH 021/138] Value list UI and telemetry (#180929) ## Add telemetry and change UI copy for value list: Screenshot 2024-04-17 at 15 09 47 Screenshot 2024-04-17 at 15 09 41 Screenshot 2024-04-17 at 15 09 30 Screenshot 2024-04-17 at 15 09 24 Screenshot 2024-04-17 at 15 09 18 --------- Co-authored-by: Nastasha Solomon <79124755+nastasha-solomon@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/common/lib/telemetry/constants.ts | 7 +++++ .../translations.ts | 2 +- .../detection_engine/rules/translations.ts | 2 +- .../components/add_list_item_popover.tsx | 4 +-- .../components/delete_list_item.tsx | 2 ++ .../inline_edit_list_item_value.tsx | 2 ++ .../value_list/components/list_item_table.tsx | 2 ++ .../components/show_value_list_modal.tsx | 6 ++++- .../components/upload_list_item.tsx | 2 ++ .../public/value_list/translations.ts | 27 +++++++++---------- .../value_lists/value_list_items.cy.ts | 2 +- 11 files changed, 37 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/public/common/lib/telemetry/constants.ts index 72c38f1cc69b5b..052a3296d84147 100644 --- a/x-pack/plugins/security_solution/public/common/lib/telemetry/constants.ts +++ b/x-pack/plugins/security_solution/public/common/lib/telemetry/constants.ts @@ -34,6 +34,13 @@ export enum TELEMETRY_EVENT { // Landing page - dashboard DASHBOARD = 'navigate_to_dashboard', CREATE_DASHBOARD = 'create_dashboard', + + // value list + OPEN_VALUE_LIST_MODAL = 'open_value_list_modal', + CREATE_VALUE_LIST_ITEM = 'create_value_list_item', + DELETE_VALUE_LIST_ITEM = 'delete_value_list_item', + EDIT_VALUE_LIST_ITEM = 'edit_value_list_item', + ADDITIONAL_UPLOAD_VALUE_LIST_ITEM = 'additinonal_upload_value_list_item', } export enum TelemetryEventTypes { diff --git a/x-pack/plugins/security_solution/public/detections/components/value_lists_management_flyout/translations.ts b/x-pack/plugins/security_solution/public/detections/components/value_lists_management_flyout/translations.ts index 37fbfaf09f6d87..caea46a2417d06 100644 --- a/x-pack/plugins/security_solution/public/detections/components/value_lists_management_flyout/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/value_lists_management_flyout/translations.ts @@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n'; export const VALUE_LISTS_FLYOUT_TITLE = i18n.translate( 'xpack.securitySolution.lists.importValueListTitle', { - defaultMessage: 'Import value lists', + defaultMessage: 'Manage value lists', } ); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts index a7823f39abb705..7ceb3fa661ba68 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts @@ -23,7 +23,7 @@ export const IMPORT_RULE = i18n.translate( export const IMPORT_VALUE_LISTS = i18n.translate( 'xpack.securitySolution.lists.detectionEngine.rules.importValueListsButton', { - defaultMessage: 'Import value lists', + defaultMessage: 'Manage value lists', } ); diff --git a/x-pack/plugins/security_solution/public/value_list/components/add_list_item_popover.tsx b/x-pack/plugins/security_solution/public/value_list/components/add_list_item_popover.tsx index 6dabd24f3f8ac2..1a0a41c602641a 100644 --- a/x-pack/plugins/security_solution/public/value_list/components/add_list_item_popover.tsx +++ b/x-pack/plugins/security_solution/public/value_list/components/add_list_item_popover.tsx @@ -23,10 +23,10 @@ import { SUCCESSFULLY_ADDED_ITEM, VALUE_REQUIRED, VALUE_LABEL, - ADD_VALUE_LIST_PLACEHOLDER, ADDING_LIST_ITEM_BUTTON, ADD_LIST_ITEM_BUTTON, } from '../translations'; +import { METRIC_TYPE, TELEMETRY_EVENT, track } from '../../common/lib/telemetry'; export const AddListItemPopover = ({ listId }: { listId: string }) => { const [isPopoverOpen, setIsPopoverOpen] = useState(false); @@ -53,6 +53,7 @@ export const AddListItemPopover = ({ listId }: { listId: string }) => { } }, onSubmit: async (values) => { + track(METRIC_TYPE.COUNT, TELEMETRY_EVENT.CREATE_VALUE_LIST_ITEM); await createListItemMutation.mutateAsync({ listId, value: values.value, http }); setIsPopoverOpen(false); formik.resetForm(); @@ -93,7 +94,6 @@ export const AddListItemPopover = ({ listId }: { listId: string }) => { name="value" icon="listAdd" data-test-subj="value-list-item-add-input" - placeholder={ADD_VALUE_LIST_PLACEHOLDER} isInvalid={!!formik.errors.value} /> diff --git a/x-pack/plugins/security_solution/public/value_list/components/delete_list_item.tsx b/x-pack/plugins/security_solution/public/value_list/components/delete_list_item.tsx index d9ad65a60cf0b7..520d9668f5ad5f 100644 --- a/x-pack/plugins/security_solution/public/value_list/components/delete_list_item.tsx +++ b/x-pack/plugins/security_solution/public/value_list/components/delete_list_item.tsx @@ -11,6 +11,7 @@ import { useDeleteListItemMutation } from '@kbn/securitysolution-list-hooks'; import { useAppToasts } from '../../common/hooks/use_app_toasts'; import { useKibana } from '../../common/lib/kibana'; import { SUCCESSFULLY_DELETED_ITEM } from '../translations'; +import { METRIC_TYPE, TELEMETRY_EVENT, track } from '../../common/lib/telemetry'; const toastOptions = { toastLifeTimeMs: 5000, @@ -32,6 +33,7 @@ export const DeleteListItem = ({ id, value }: { id: string; value: string }) => }); const deleteListItem = useCallback(() => { + track(METRIC_TYPE.COUNT, TELEMETRY_EVENT.DELETE_VALUE_LIST_ITEM); deleteListItemMutation.mutate({ id, http }); }, [deleteListItemMutation, id, http]); diff --git a/x-pack/plugins/security_solution/public/value_list/components/inline_edit_list_item_value.tsx b/x-pack/plugins/security_solution/public/value_list/components/inline_edit_list_item_value.tsx index 4ef639bcdbebbb..d8777a8d83d00b 100644 --- a/x-pack/plugins/security_solution/public/value_list/components/inline_edit_list_item_value.tsx +++ b/x-pack/plugins/security_solution/public/value_list/components/inline_edit_list_item_value.tsx @@ -12,6 +12,7 @@ import { usePatchListItemMutation } from '@kbn/securitysolution-list-hooks'; import { useAppToasts } from '../../common/hooks/use_app_toasts'; import { useKibana } from '../../common/lib/kibana/kibana_react'; import { EDIT_TEXT_INLINE_LABEL, SUCCESSFULLY_UPDATED_LIST_ITEM } from '../translations'; +import { METRIC_TYPE, TELEMETRY_EVENT, track } from '../../common/lib/telemetry'; const toastOptions = { toastLifeTimeMs: 5000, @@ -41,6 +42,7 @@ export const InlineEditListItemValue = ({ listItem }: { listItem: ListItemSchema const onSave = useCallback( async (newValue) => { + track(METRIC_TYPE.COUNT, TELEMETRY_EVENT.EDIT_VALUE_LIST_ITEM); await patchListItemMutation.mutateAsync({ id: listItem.id, value: newValue, diff --git a/x-pack/plugins/security_solution/public/value_list/components/list_item_table.tsx b/x-pack/plugins/security_solution/public/value_list/components/list_item_table.tsx index 50226ea047acc6..11ecd486aa452f 100644 --- a/x-pack/plugins/security_solution/public/value_list/components/list_item_table.tsx +++ b/x-pack/plugins/security_solution/public/value_list/components/list_item_table.tsx @@ -20,6 +20,7 @@ import { FAILED_TO_FETCH_LIST_ITEM, DELETE_LIST_ITEM, DELETE_LIST_ITEM_DESCRIPTION, + NOT_FOUND_ITEMS, } from '../translations'; export const ListItemTable = ({ @@ -80,6 +81,7 @@ export const ListItemTable = ({ error={isError ? FAILED_TO_FETCH_LIST_ITEM : undefined} loading={loading} onChange={onChange} + noItemsMessage={NOT_FOUND_ITEMS} /> ); }; diff --git a/x-pack/plugins/security_solution/public/value_list/components/show_value_list_modal.tsx b/x-pack/plugins/security_solution/public/value_list/components/show_value_list_modal.tsx index 0cf0608425b2f9..96ab0ab599ee4a 100644 --- a/x-pack/plugins/security_solution/public/value_list/components/show_value_list_modal.tsx +++ b/x-pack/plugins/security_solution/public/value_list/components/show_value_list_modal.tsx @@ -10,6 +10,7 @@ import React, { useState, useCallback } from 'react'; import { useListsPrivileges } from '../../detections/containers/detection_engine/lists/use_lists_privileges'; import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; import { ValueListModal } from './value_list_modal'; +import { METRIC_TYPE, TELEMETRY_EVENT, track } from '../../common/lib/telemetry'; export const ShowValueListModal = ({ listId, @@ -27,7 +28,10 @@ export const ShowValueListModal = ({ ); const onCloseModal = useCallback(() => setShowModal(false), []); - const onShowModal = useCallback(() => setShowModal(true), []); + const onShowModal = useCallback(() => { + track(METRIC_TYPE.CLICK, TELEMETRY_EVENT.OPEN_VALUE_LIST_MODAL); + setShowModal(true); + }, []); if (loading) return null; diff --git a/x-pack/plugins/security_solution/public/value_list/components/upload_list_item.tsx b/x-pack/plugins/security_solution/public/value_list/components/upload_list_item.tsx index 7e43629a6578c6..3fe12339fe0955 100644 --- a/x-pack/plugins/security_solution/public/value_list/components/upload_list_item.tsx +++ b/x-pack/plugins/security_solution/public/value_list/components/upload_list_item.tsx @@ -20,6 +20,7 @@ import { FAILED_TO_UPLOAD_LIST_ITEM, SUCCESSFULY_UPLOAD_LIST_ITEMS, } from '../translations'; +import { METRIC_TYPE, TELEMETRY_EVENT, track } from '../../common/lib/telemetry'; const validFileTypes = ['text/csv', 'text/plain']; @@ -45,6 +46,7 @@ export const UploadListItem = ({ listId, type }: { listId: string; type: ListTyp const handleImport = useCallback(() => { if (!importState.loading && file) { ctrl.current = new AbortController(); + track(METRIC_TYPE.COUNT, TELEMETRY_EVENT.ADDITIONAL_UPLOAD_VALUE_LIST_ITEM); importList({ file, listId, diff --git a/x-pack/plugins/security_solution/public/value_list/translations.ts b/x-pack/plugins/security_solution/public/value_list/translations.ts index 62b81fa61cab25..d97ece63dbe15e 100644 --- a/x-pack/plugins/security_solution/public/value_list/translations.ts +++ b/x-pack/plugins/security_solution/public/value_list/translations.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; export const ADD_LIST_ITEM = i18n.translate('xpack.securitySolution.listItems.addListItem', { - defaultMessage: 'Add list item', + defaultMessage: 'Create list item', }); export const SUCCESSFULLY_ADDED_ITEM = i18n.translate( @@ -23,20 +23,13 @@ export const VALUE_REQUIRED = i18n.translate('xpack.securitySolution.listItems.v }); export const VALUE_LABEL = i18n.translate('xpack.securitySolution.listItems.valueLabel', { - defaultMessage: 'Value', + defaultMessage: 'Enter a new value for the list', }); -export const ADD_VALUE_LIST_PLACEHOLDER = i18n.translate( - 'xpack.securitySolution.listItems.addValueListPlaceholder', - { - defaultMessage: 'Add list item..', - } -); - export const ADD_LIST_ITEM_BUTTON = i18n.translate( 'xpack.securitySolution.listItems.addListItemButton', { - defaultMessage: 'Add', + defaultMessage: 'Add list item', } ); @@ -75,19 +68,19 @@ export const COLUMN_VALUE = i18n.translate('xpack.securitySolution.listItems.col export const COLUMN_UPDATED_AT = i18n.translate( 'xpack.securitySolution.listItems.columnUpdatedAt', { - defaultMessage: 'Updated At', + defaultMessage: 'Updated at', } ); export const COLUMN_UPDATED_BY = i18n.translate( 'xpack.securitySolution.listItems.columnUpdatedBy', { - defaultMessage: 'Updated By', + defaultMessage: 'Updated by', } ); export const COLUMN_ACTIONS = i18n.translate('xpack.securitySolution.listItems.columnActions', { - defaultMessage: 'Actions', + defaultMessage: 'Action', }); export const FAILED_TO_FETCH_LIST_ITEM = i18n.translate( @@ -131,7 +124,7 @@ export const FAILED_TO_UPLOAD_LIST_ITEM_TITLE = i18n.translate( ); export const UPLOAD_TOOLTIP = i18n.translate('xpack.securitySolution.listItems.uploadTooltip', { - defaultMessage: 'All items from the file will be added as new items', + defaultMessage: 'All items from the file will be added to the value list.', }); export const UPLOAD_FILE_PICKER_INITAL_PROMT_TEXT = i18n.translate( @@ -163,6 +156,10 @@ export const INFO_TOTAL_ITEMS = i18n.translate('xpack.securitySolution.listItems export const getInfoTotalItems = (listType: string) => i18n.translate('xpack.securitySolution.listItems.searchBar', { - defaultMessage: 'Filter your data using KQL syntax - {listType}:*', + defaultMessage: 'Filter your data using KQL syntax, for example: {listType}:*', values: { listType }, }); + +export const NOT_FOUND_ITEMS = i18n.translate('xpack.securitySolution.listItems.notFoundItems', { + defaultMessage: '0 list items match your search criteria.', +}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/value_lists/value_list_items.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/value_lists/value_list_items.cy.ts index c8fd74fa64397d..79933ea4d7f339 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/value_lists/value_list_items.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/value_lists/value_list_items.cy.ts @@ -88,7 +88,7 @@ describe( getValueListItemsTableRow().should('have.length', perPage); searchValueListItemsModal('keyword:not_exists'); getValueListItemsTableRow().should('have.length', 1); - cy.get(VALUE_LIST_ITEMS_MODAL_TABLE).contains('No items found'); + cy.get(VALUE_LIST_ITEMS_MODAL_TABLE).contains('0 list items match your search criteria.'); searchValueListItemsModal('keyword:*or*'); getValueListItemsTableRow().should('have.length', 4); From f2f5ec927c8439b536e874ff04e081ec95758fad Mon Sep 17 00:00:00 2001 From: Jon Date: Wed, 24 Apr 2024 10:26:11 -0500 Subject: [PATCH 022/138] [build/serverless] Do not use spot instances (#181578) This build step doesn't support retries, if the docker image has already been uploaded once it exits early. We want to rule out spot preemptions as a cause of failure. --- .buildkite/pipelines/artifacts_container_image.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.buildkite/pipelines/artifacts_container_image.yml b/.buildkite/pipelines/artifacts_container_image.yml index 972fada9ddde2d..63744d64aba92f 100644 --- a/.buildkite/pipelines/artifacts_container_image.yml +++ b/.buildkite/pipelines/artifacts_container_image.yml @@ -2,9 +2,5 @@ steps: - command: .buildkite/scripts/steps/artifacts/docker_image.sh label: Build serverless container images agents: - queue: n2-16-spot + queue: c2-16 timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 From 4cf38adac29c7a65ab142ba428cdc3ae0648f2c4 Mon Sep 17 00:00:00 2001 From: "Eyo O. Eyo" <7893459+eokoneyo@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:30:35 +0100 Subject: [PATCH 023/138] [ES|QL] Multiline query in expanded mode shows undefined for line number (#181544) ## Summary closes https://github.com/elastic/kibana/issues/180570 Thanks to @drewdaemon for the preliminary investigation to the root cause of the issue, see https://github.com/elastic/kibana/issues/180570#issuecomment-2059590022. Building off of this, I noticed there's actually a first render pass where the code editor renders just fine, but for some reason it renders again ~causing~ which in turn causes the issue to happen see Screenshot 2024-04-24 at 01 48 18 From the image above we see that the editor has it's line numbers rendered appropriately but then there's another render triggered which causes the issue, we'll notice that the dynamic overlay instance in the image above have their property of `_shouldRender` marked as false, this is because a render already happened see line 97 from the same image where said dynamic overlay is marked as rendered; Screenshot 2024-04-24 at 09 09 51 See [here](https://github.com/microsoft/vscode/blob/a3944f74adb303047355e8c7d4b401bfba4e1a0d/src/vs/editor/common/viewEventHandler.ts#L30-L32) for the definition of `onDidRender`. This then makes it such that for the next render pass the render result isn't available. My assumption is that because we are updating the layout value of the editor, on every model change without actually taking into consideration the state of the editor it causes the editor to go out of sync, see Screenshot 2024-04-24 at 09 31 45 To counteract this, this PR introduces a guard that only updates the editor height when the event received actually has a diff in `contentHeight` property. Alongside this computation of the editor height is now inferred by reading values provided by the editor itself and not the DOM element of the editor. ## Visuals https://github.com/elastic/kibana/assets/7893459/26bb60ad-9ea1-41fe-854a-5c8a7be2e29d --- .../src/text_based_languages_editor.styles.ts | 2 +- .../src/text_based_languages_editor.tsx | 45 ++++++++----------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.styles.ts b/packages/kbn-text-based-editor/src/text_based_languages_editor.styles.ts index 8d8bd72eb8dcd6..8e5fbb1974a55a 100644 --- a/packages/kbn-text-based-editor/src/text_based_languages_editor.styles.ts +++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.styles.ts @@ -12,7 +12,7 @@ export const EDITOR_INITIAL_HEIGHT_EXPANDED = 140; export const EDITOR_MIN_HEIGHT = 40; export const EDITOR_MAX_HEIGHT = 400; -export const textBasedLanguagedEditorStyles = ( +export const textBasedLanguageEditorStyles = ( euiTheme: EuiThemeComputed, isCompactFocused: boolean, editorHeight: number, diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx index cf026c79fbc55d..7cbf3a5a5cbf4b 100644 --- a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx +++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx @@ -43,7 +43,7 @@ import { import { CodeEditor, CodeEditorProps } from '@kbn/code-editor'; import { - textBasedLanguagedEditorStyles, + textBasedLanguageEditorStyles, EDITOR_INITIAL_HEIGHT, EDITOR_INITIAL_HEIGHT_EXPANDED, EDITOR_MAX_HEIGHT, @@ -189,7 +189,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ kibana.services; const timeZone = core?.uiSettings?.get('dateFormat:tz'); const [code, setCode] = useState(queryString ?? ''); - const [codeOneLiner, setCodeOneLiner] = useState(''); + const [codeOneLiner, setCodeOneLiner] = useState(null); // To make server side errors less "sticky", register the state of the code when submitting const [codeWhenSubmitted, setCodeStateOnSubmission] = useState(code); const [editorHeight, setEditorHeight] = useState( @@ -273,7 +273,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ }); }); - const styles = textBasedLanguagedEditorStyles( + const styles = textBasedLanguageEditorStyles( euiTheme, isCompactFocused, editorHeight, @@ -342,34 +342,24 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ updateLinesFromModel = false; clickedOutside = true; if (editor1.current) { - const editorElement = editor1.current.getDomNode(); - if (editorElement) { - editorElement.style.height = `${EDITOR_INITIAL_HEIGHT}px`; - const contentWidth = Number(editorElement?.style.width.replace('px', '')); - calculateVisibleCode(contentWidth, true); - editor1.current.layout({ width: contentWidth, height: EDITOR_INITIAL_HEIGHT }); - } + const contentWidth = editor1.current.getLayoutInfo().width; + calculateVisibleCode(contentWidth, true); + editor1.current.layout({ width: contentWidth, height: EDITOR_INITIAL_HEIGHT }); } }; const updateHeight = useCallback((editor: monaco.editor.IStandaloneCodeEditor) => { - if (lines === 1 || clickedOutside || initialRender) return; - const editorElement = editor.getDomNode(); + if (clickedOutside || initialRender) return; const contentHeight = Math.min(MAX_COMPACT_VIEW_LENGTH, editor.getContentHeight()); - - if (editorElement) { - editorElement.style.height = `${contentHeight}px`; - } - const contentWidth = Number(editorElement?.style.width.replace('px', '')); - editor.layout({ width: contentWidth, height: contentHeight }); setEditorHeight(contentHeight); + editor.layout({ width: editor.getLayoutInfo().width, height: contentHeight }); }, []); const onEditorFocus = useCallback(() => { setIsCompactFocused(true); setIsCodeEditorExpandedFocused(true); setShowLineNumbers(true); - setCodeOneLiner(''); + setCodeOneLiner(null); clickedOutside = false; initialRender = false; updateLinesFromModel = true; @@ -598,13 +588,9 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ useEffect(() => { if (editor1.current && !isCompactFocused) { - const editorElement = editor1.current.getDomNode(); - if (editorElement) { - const contentWidth = Number(editorElement?.style.width.replace('px', '')); - if (code !== queryString) { - setCode(queryString); - calculateVisibleCode(contentWidth); - } + if (code !== queryString) { + setCode(queryString); + calculateVisibleCode(editor1.current.getLayoutInfo().width); } } }, [calculateVisibleCode, code, isCompactFocused, queryString]); @@ -944,9 +930,14 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, onQuerySubmit ); + if (!isCodeEditorExpanded) { editor.onDidContentSizeChange((e) => { - updateHeight(editor); + // @ts-expect-error the property _oldContentHeight exists on the event object received but + // is not available on the type definition + if (e.contentHeight !== e._oldContentHeight) { + updateHeight(editor); + } }); } }} From 4bfe566c18f6c6ef7e347c2a4e0d59b08da6ad51 Mon Sep 17 00:00:00 2001 From: Kevin Delemme Date: Wed, 24 Apr 2024 11:53:39 -0400 Subject: [PATCH 024/138] Use internal formatErrors (#181065) --- .../src/decode_request_params.test.ts | 3 ++- .../src/decode_request_params.ts | 11 +++++------ packages/kbn-server-route-repository/tsconfig.json | 1 - .../apm_routes/register_apm_server_routes.test.ts | 3 ++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/kbn-server-route-repository/src/decode_request_params.test.ts b/packages/kbn-server-route-repository/src/decode_request_params.test.ts index 8f7e6f421cd366..0cb8d280f28e47 100644 --- a/packages/kbn-server-route-repository/src/decode_request_params.test.ts +++ b/packages/kbn-server-route-repository/src/decode_request_params.test.ts @@ -69,7 +69,8 @@ describe('decodeRequestParams', () => { }; expect(decode).toThrowErrorMatchingInlineSnapshot(` - "Excess keys are not allowed: + "Failed to validate: + Excess keys are not allowed: path.extraKey" `); }); diff --git a/packages/kbn-server-route-repository/src/decode_request_params.ts b/packages/kbn-server-route-repository/src/decode_request_params.ts index e9b75ded73d016..0893524a3f9e95 100644 --- a/packages/kbn-server-route-repository/src/decode_request_params.ts +++ b/packages/kbn-server-route-repository/src/decode_request_params.ts @@ -5,12 +5,11 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import * as t from 'io-ts'; -import { omitBy, isPlainObject, isEmpty } from 'lodash'; -import { isLeft } from 'fp-ts/lib/Either'; import Boom from '@hapi/boom'; -import { strictKeysRt } from '@kbn/io-ts-utils'; -import { formatErrors } from '@kbn/securitysolution-io-ts-utils'; +import { formatErrors, strictKeysRt } from '@kbn/io-ts-utils'; +import { isLeft } from 'fp-ts/lib/Either'; +import * as t from 'io-ts'; +import { isEmpty, isPlainObject, omitBy } from 'lodash'; import { RouteParamsRT } from './typings'; interface KibanaRequestParams { @@ -36,7 +35,7 @@ export function decodeRequestParams( const result = strictKeysRt(paramsRt).decode(paramMap); if (isLeft(result)) { - throw Boom.badRequest(formatErrors(result.left).join('|')); + throw Boom.badRequest(formatErrors(result.left)); } return result.right; diff --git a/packages/kbn-server-route-repository/tsconfig.json b/packages/kbn-server-route-repository/tsconfig.json index f5f84f5114b7d5..67a5631bca59e0 100644 --- a/packages/kbn-server-route-repository/tsconfig.json +++ b/packages/kbn-server-route-repository/tsconfig.json @@ -18,7 +18,6 @@ "@kbn/core-http-server", "@kbn/core-lifecycle-server", "@kbn/logging", - "@kbn/securitysolution-io-ts-utils" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/observability_solution/apm/server/routes/apm_routes/register_apm_server_routes.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/apm_routes/register_apm_server_routes.test.ts index 47ada9ee20f4c1..c5e652fa910d08 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/apm_routes/register_apm_server_routes.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/apm_routes/register_apm_server_routes.test.ts @@ -256,7 +256,8 @@ describe('createApi', () => { expect(response.custom).toHaveBeenCalledWith({ body: { attributes: { _inspect: [], data: null }, - message: 'Invalid value "1" supplied to "query,_inspect"', + message: `Failed to validate: + in /query/_inspect: 1 does not match expected type pipe(JSON, boolean)`, }, statusCode: 400, }); From ed2f63bf7e540d63229dfc32dc6d8f6a6b895485 Mon Sep 17 00:00:00 2001 From: Julia Rechkunova Date: Wed, 24 Apr 2024 18:18:42 +0200 Subject: [PATCH 025/138] [Discover] Split functional tests in more groups (#180046) ## Summary This PR splits existing functional tests into more groups so each of them finishes running quicker. - `group1` into `group1` and `group5` - `group2` into `group2_data_grid1`, `group2_data_grid2`, `group2_data_grid3` - `group3` into `group3` and `group6` - `group4` into `group4`, `group7` and `group8` (almost empty) --------- Co-authored-by: Davis McPhee --- .buildkite/ftr_configs.yml | 8 ++++- test/functional/apps/discover/group1/index.ts | 10 ------ .../_data_grid.ts | 0 .../_data_grid_context.ts | 0 .../_data_grid_copy_to_clipboard.ts | 0 .../_data_grid_doc_navigation.ts | 0 .../_data_grid_doc_table.ts | 0 .../{group2 => group2_data_grid1}/config.ts | 0 .../{group2 => group2_data_grid1}/index.ts | 10 +----- .../_data_grid_field_data.ts | 0 .../_data_grid_field_tokens.ts | 0 .../_data_grid_footer.ts | 0 .../_data_grid_new_line.ts | 0 .../apps/discover/group2_data_grid2/config.ts | 18 ++++++++++ .../apps/discover/group2_data_grid2/index.ts | 28 +++++++++++++++ .../_data_grid_pagination.ts | 0 .../_data_grid_row_height.ts | 0 .../_data_grid_row_navigation.ts | 0 .../_data_grid_sample_size.ts | 0 .../apps/discover/group2_data_grid3/config.ts | 18 ++++++++++ .../apps/discover/group2_data_grid3/index.ts | 28 +++++++++++++++ test/functional/apps/discover/group3/index.ts | 5 --- test/functional/apps/discover/group4/index.ts | 8 ----- .../{group1 => group5}/_field_data.ts | 0 .../_field_data_with_fields_api.ts | 0 .../{group1 => group5}/_filter_editor.ts | 0 .../{group1 => group5}/_greeting_screen.ts | 0 .../discover/{group1 => group5}/_inspector.ts | 0 .../{group1 => group5}/_large_string.ts | 0 .../discover/{group1 => group5}/_no_data.ts | 0 .../{group1 => group5}/_shared_links.ts | 0 .../{group1 => group5}/_source_filters.ts | 0 .../discover/{group1 => group5}/_url_state.ts | 0 .../functional/apps/discover/group5/config.ts | 18 ++++++++++ test/functional/apps/discover/group5/index.ts | 34 +++++++++++++++++++ .../discover/{group3 => group6}/_sidebar.ts | 0 .../_sidebar_field_stats.ts | 0 .../{group3 => group6}/_time_field_column.ts | 0 .../_unsaved_changes_badge.ts | 0 .../{group3 => group6}/_view_mode_toggle.ts | 0 .../functional/apps/discover/group6/config.ts | 18 ++++++++++ test/functional/apps/discover/group6/index.ts | 29 ++++++++++++++++ .../{group4 => group7}/_huge_fields.ts | 0 .../_indexpattern_with_unmapped_fields.ts | 0 .../_indexpattern_without_timefield.ts | 0 .../{group4 => group7}/_new_search.ts | 0 .../_request_cancellation.ts | 0 .../_runtime_fields_editor.ts | 0 .../_search_on_page_load.ts | 0 .../functional/apps/discover/group7/config.ts | 18 ++++++++++ test/functional/apps/discover/group7/index.ts | 31 +++++++++++++++++ .../{group4 => group8}/_hide_announcements.ts | 0 .../functional/apps/discover/group8/config.ts | 18 ++++++++++ test/functional/apps/discover/group8/index.ts | 25 ++++++++++++++ test/functional/firefox/discover.config.ts | 8 ++++- .../common/discover/group1/index.ts | 1 - .../common/discover/group2/index.ts | 1 - .../common/discover/group3/index.ts | 2 -- .../{group2 => group4}/_adhoc_data_views.ts | 0 .../common/discover/group4/index.ts | 25 ++++++++++++++ .../discover/{group1 => group5}/_url_state.ts | 0 .../common/discover/group5/index.ts | 25 ++++++++++++++ .../discover/{group3 => group6}/_sidebar.ts | 0 .../_unsaved_changes_badge.ts | 0 .../common/discover/group6/index.ts | 26 ++++++++++++++ .../common_configs/config.group5.ts | 3 ++ .../search/common_configs/config.group5.ts | 3 ++ .../security/common_configs/config.group5.ts | 3 ++ 68 files changed, 383 insertions(+), 38 deletions(-) rename test/functional/apps/discover/{group2 => group2_data_grid1}/_data_grid.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid1}/_data_grid_context.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid1}/_data_grid_copy_to_clipboard.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid1}/_data_grid_doc_navigation.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid1}/_data_grid_doc_table.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid1}/config.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid1}/index.ts (67%) rename test/functional/apps/discover/{group2 => group2_data_grid2}/_data_grid_field_data.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid2}/_data_grid_field_tokens.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid2}/_data_grid_footer.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid2}/_data_grid_new_line.ts (100%) create mode 100644 test/functional/apps/discover/group2_data_grid2/config.ts create mode 100644 test/functional/apps/discover/group2_data_grid2/index.ts rename test/functional/apps/discover/{group2 => group2_data_grid3}/_data_grid_pagination.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid3}/_data_grid_row_height.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid3}/_data_grid_row_navigation.ts (100%) rename test/functional/apps/discover/{group2 => group2_data_grid3}/_data_grid_sample_size.ts (100%) create mode 100644 test/functional/apps/discover/group2_data_grid3/config.ts create mode 100644 test/functional/apps/discover/group2_data_grid3/index.ts rename test/functional/apps/discover/{group1 => group5}/_field_data.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_field_data_with_fields_api.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_filter_editor.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_greeting_screen.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_inspector.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_large_string.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_no_data.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_shared_links.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_source_filters.ts (100%) rename test/functional/apps/discover/{group1 => group5}/_url_state.ts (100%) create mode 100644 test/functional/apps/discover/group5/config.ts create mode 100644 test/functional/apps/discover/group5/index.ts rename test/functional/apps/discover/{group3 => group6}/_sidebar.ts (100%) rename test/functional/apps/discover/{group3 => group6}/_sidebar_field_stats.ts (100%) rename test/functional/apps/discover/{group3 => group6}/_time_field_column.ts (100%) rename test/functional/apps/discover/{group3 => group6}/_unsaved_changes_badge.ts (100%) rename test/functional/apps/discover/{group3 => group6}/_view_mode_toggle.ts (100%) create mode 100644 test/functional/apps/discover/group6/config.ts create mode 100644 test/functional/apps/discover/group6/index.ts rename test/functional/apps/discover/{group4 => group7}/_huge_fields.ts (100%) rename test/functional/apps/discover/{group4 => group7}/_indexpattern_with_unmapped_fields.ts (100%) rename test/functional/apps/discover/{group4 => group7}/_indexpattern_without_timefield.ts (100%) rename test/functional/apps/discover/{group4 => group7}/_new_search.ts (100%) rename test/functional/apps/discover/{group4 => group7}/_request_cancellation.ts (100%) rename test/functional/apps/discover/{group4 => group7}/_runtime_fields_editor.ts (100%) rename test/functional/apps/discover/{group4 => group7}/_search_on_page_load.ts (100%) create mode 100644 test/functional/apps/discover/group7/config.ts create mode 100644 test/functional/apps/discover/group7/index.ts rename test/functional/apps/discover/{group4 => group8}/_hide_announcements.ts (100%) create mode 100644 test/functional/apps/discover/group8/config.ts create mode 100644 test/functional/apps/discover/group8/index.ts rename x-pack/test_serverless/functional/test_suites/common/discover/{group2 => group4}/_adhoc_data_views.ts (100%) create mode 100644 x-pack/test_serverless/functional/test_suites/common/discover/group4/index.ts rename x-pack/test_serverless/functional/test_suites/common/discover/{group1 => group5}/_url_state.ts (100%) create mode 100644 x-pack/test_serverless/functional/test_suites/common/discover/group5/index.ts rename x-pack/test_serverless/functional/test_suites/common/discover/{group3 => group6}/_sidebar.ts (100%) rename x-pack/test_serverless/functional/test_suites/common/discover/{group3 => group6}/_unsaved_changes_badge.ts (100%) create mode 100644 x-pack/test_serverless/functional/test_suites/common/discover/group6/index.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index a4c7c64762299f..b17a63170f907a 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -115,9 +115,15 @@ enabled: - test/functional/apps/discover/classic/config.ts - test/functional/apps/discover/embeddable/config.ts - test/functional/apps/discover/group1/config.ts - - test/functional/apps/discover/group2/config.ts + - test/functional/apps/discover/group2_data_grid1/config.ts + - test/functional/apps/discover/group2_data_grid2/config.ts + - test/functional/apps/discover/group2_data_grid3/config.ts - test/functional/apps/discover/group3/config.ts - test/functional/apps/discover/group4/config.ts + - test/functional/apps/discover/group5/config.ts + - test/functional/apps/discover/group6/config.ts + - test/functional/apps/discover/group7/config.ts + - test/functional/apps/discover/group8/config.ts - test/functional/apps/getting_started/config.ts - test/functional/apps/home/config.ts - test/functional/apps/kibana_overview/config.ts diff --git a/test/functional/apps/discover/group1/index.ts b/test/functional/apps/discover/group1/index.ts index 375954797c3caa..2ca6413a11c220 100644 --- a/test/functional/apps/discover/group1/index.ts +++ b/test/functional/apps/discover/group1/index.ts @@ -20,23 +20,13 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); }); - loadTestFile(require.resolve('./_no_data')); loadTestFile(require.resolve('./_discover')); loadTestFile(require.resolve('./_discover_accessibility')); loadTestFile(require.resolve('./_discover_histogram_breakdown')); loadTestFile(require.resolve('./_discover_histogram')); loadTestFile(require.resolve('./_doc_accessibility')); - loadTestFile(require.resolve('./_filter_editor')); loadTestFile(require.resolve('./_errors')); - loadTestFile(require.resolve('./_field_data')); - loadTestFile(require.resolve('./_field_data_with_fields_api')); - loadTestFile(require.resolve('./_shared_links')); - loadTestFile(require.resolve('./_source_filters')); - loadTestFile(require.resolve('./_large_string')); - loadTestFile(require.resolve('./_greeting_screen')); - loadTestFile(require.resolve('./_inspector')); loadTestFile(require.resolve('./_date_nanos')); loadTestFile(require.resolve('./_date_nanos_mixed')); - loadTestFile(require.resolve('./_url_state')); }); } diff --git a/test/functional/apps/discover/group2/_data_grid.ts b/test/functional/apps/discover/group2_data_grid1/_data_grid.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid.ts rename to test/functional/apps/discover/group2_data_grid1/_data_grid.ts diff --git a/test/functional/apps/discover/group2/_data_grid_context.ts b/test/functional/apps/discover/group2_data_grid1/_data_grid_context.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_context.ts rename to test/functional/apps/discover/group2_data_grid1/_data_grid_context.ts diff --git a/test/functional/apps/discover/group2/_data_grid_copy_to_clipboard.ts b/test/functional/apps/discover/group2_data_grid1/_data_grid_copy_to_clipboard.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_copy_to_clipboard.ts rename to test/functional/apps/discover/group2_data_grid1/_data_grid_copy_to_clipboard.ts diff --git a/test/functional/apps/discover/group2/_data_grid_doc_navigation.ts b/test/functional/apps/discover/group2_data_grid1/_data_grid_doc_navigation.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_doc_navigation.ts rename to test/functional/apps/discover/group2_data_grid1/_data_grid_doc_navigation.ts diff --git a/test/functional/apps/discover/group2/_data_grid_doc_table.ts b/test/functional/apps/discover/group2_data_grid1/_data_grid_doc_table.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_doc_table.ts rename to test/functional/apps/discover/group2_data_grid1/_data_grid_doc_table.ts diff --git a/test/functional/apps/discover/group2/config.ts b/test/functional/apps/discover/group2_data_grid1/config.ts similarity index 100% rename from test/functional/apps/discover/group2/config.ts rename to test/functional/apps/discover/group2_data_grid1/config.ts diff --git a/test/functional/apps/discover/group2/index.ts b/test/functional/apps/discover/group2_data_grid1/index.ts similarity index 67% rename from test/functional/apps/discover/group2/index.ts rename to test/functional/apps/discover/group2_data_grid1/index.ts index 2639601e12c171..c9e9e9739337a6 100644 --- a/test/functional/apps/discover/group2/index.ts +++ b/test/functional/apps/discover/group2_data_grid1/index.ts @@ -11,7 +11,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const browser = getService('browser'); - describe('discover/group2', function () { + describe('discover/group2/data_grid1', function () { before(async function () { await browser.setWindowSize(1600, 1200); }); @@ -22,16 +22,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_data_grid')); loadTestFile(require.resolve('./_data_grid_context')); - loadTestFile(require.resolve('./_data_grid_field_data')); loadTestFile(require.resolve('./_data_grid_doc_navigation')); - loadTestFile(require.resolve('./_data_grid_row_navigation')); loadTestFile(require.resolve('./_data_grid_doc_table')); loadTestFile(require.resolve('./_data_grid_copy_to_clipboard')); - loadTestFile(require.resolve('./_data_grid_row_height')); - loadTestFile(require.resolve('./_data_grid_new_line')); - loadTestFile(require.resolve('./_data_grid_sample_size')); - loadTestFile(require.resolve('./_data_grid_pagination')); - loadTestFile(require.resolve('./_data_grid_footer')); - loadTestFile(require.resolve('./_data_grid_field_tokens')); }); } diff --git a/test/functional/apps/discover/group2/_data_grid_field_data.ts b/test/functional/apps/discover/group2_data_grid2/_data_grid_field_data.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_field_data.ts rename to test/functional/apps/discover/group2_data_grid2/_data_grid_field_data.ts diff --git a/test/functional/apps/discover/group2/_data_grid_field_tokens.ts b/test/functional/apps/discover/group2_data_grid2/_data_grid_field_tokens.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_field_tokens.ts rename to test/functional/apps/discover/group2_data_grid2/_data_grid_field_tokens.ts diff --git a/test/functional/apps/discover/group2/_data_grid_footer.ts b/test/functional/apps/discover/group2_data_grid2/_data_grid_footer.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_footer.ts rename to test/functional/apps/discover/group2_data_grid2/_data_grid_footer.ts diff --git a/test/functional/apps/discover/group2/_data_grid_new_line.ts b/test/functional/apps/discover/group2_data_grid2/_data_grid_new_line.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_new_line.ts rename to test/functional/apps/discover/group2_data_grid2/_data_grid_new_line.ts diff --git a/test/functional/apps/discover/group2_data_grid2/config.ts b/test/functional/apps/discover/group2_data_grid2/config.ts new file mode 100644 index 00000000000000..a70a190ca63f81 --- /dev/null +++ b/test/functional/apps/discover/group2_data_grid2/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/test/functional/apps/discover/group2_data_grid2/index.ts b/test/functional/apps/discover/group2_data_grid2/index.ts new file mode 100644 index 00000000000000..1d3736cafe80b2 --- /dev/null +++ b/test/functional/apps/discover/group2_data_grid2/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group2/data_grid2', function () { + before(async function () { + await browser.setWindowSize(1600, 1200); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_data_grid_new_line')); + loadTestFile(require.resolve('./_data_grid_footer')); + loadTestFile(require.resolve('./_data_grid_field_data')); + loadTestFile(require.resolve('./_data_grid_field_tokens')); + }); +} diff --git a/test/functional/apps/discover/group2/_data_grid_pagination.ts b/test/functional/apps/discover/group2_data_grid3/_data_grid_pagination.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_pagination.ts rename to test/functional/apps/discover/group2_data_grid3/_data_grid_pagination.ts diff --git a/test/functional/apps/discover/group2/_data_grid_row_height.ts b/test/functional/apps/discover/group2_data_grid3/_data_grid_row_height.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_row_height.ts rename to test/functional/apps/discover/group2_data_grid3/_data_grid_row_height.ts diff --git a/test/functional/apps/discover/group2/_data_grid_row_navigation.ts b/test/functional/apps/discover/group2_data_grid3/_data_grid_row_navigation.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_row_navigation.ts rename to test/functional/apps/discover/group2_data_grid3/_data_grid_row_navigation.ts diff --git a/test/functional/apps/discover/group2/_data_grid_sample_size.ts b/test/functional/apps/discover/group2_data_grid3/_data_grid_sample_size.ts similarity index 100% rename from test/functional/apps/discover/group2/_data_grid_sample_size.ts rename to test/functional/apps/discover/group2_data_grid3/_data_grid_sample_size.ts diff --git a/test/functional/apps/discover/group2_data_grid3/config.ts b/test/functional/apps/discover/group2_data_grid3/config.ts new file mode 100644 index 00000000000000..a70a190ca63f81 --- /dev/null +++ b/test/functional/apps/discover/group2_data_grid3/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/test/functional/apps/discover/group2_data_grid3/index.ts b/test/functional/apps/discover/group2_data_grid3/index.ts new file mode 100644 index 00000000000000..7200eb1e9bf106 --- /dev/null +++ b/test/functional/apps/discover/group2_data_grid3/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group2/data_grid3', function () { + before(async function () { + await browser.setWindowSize(1600, 1200); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_data_grid_row_navigation')); + loadTestFile(require.resolve('./_data_grid_row_height')); + loadTestFile(require.resolve('./_data_grid_sample_size')); + loadTestFile(require.resolve('./_data_grid_pagination')); + }); +} diff --git a/test/functional/apps/discover/group3/index.ts b/test/functional/apps/discover/group3/index.ts index a80ae44e498017..582710e419a755 100644 --- a/test/functional/apps/discover/group3/index.ts +++ b/test/functional/apps/discover/group3/index.ts @@ -21,14 +21,9 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { }); loadTestFile(require.resolve('./_default_columns')); - loadTestFile(require.resolve('./_time_field_column')); loadTestFile(require.resolve('./_drag_drop')); - loadTestFile(require.resolve('./_sidebar')); - loadTestFile(require.resolve('./_sidebar_field_stats')); loadTestFile(require.resolve('./_request_counts')); loadTestFile(require.resolve('./_doc_viewer')); - loadTestFile(require.resolve('./_view_mode_toggle')); - loadTestFile(require.resolve('./_unsaved_changes_badge')); loadTestFile(require.resolve('./_panels_toggle')); loadTestFile(require.resolve('./_lens_vis')); }); diff --git a/test/functional/apps/discover/group4/index.ts b/test/functional/apps/discover/group4/index.ts index 4a145b06cf2487..211b4501ed329c 100644 --- a/test/functional/apps/discover/group4/index.ts +++ b/test/functional/apps/discover/group4/index.ts @@ -20,22 +20,14 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); }); - loadTestFile(require.resolve('./_indexpattern_without_timefield')); loadTestFile(require.resolve('./_discover_fields_api')); loadTestFile(require.resolve('./_adhoc_data_views')); loadTestFile(require.resolve('./_esql_view')); - loadTestFile(require.resolve('./_indexpattern_with_unmapped_fields')); - loadTestFile(require.resolve('./_runtime_fields_editor')); - loadTestFile(require.resolve('./_huge_fields')); loadTestFile(require.resolve('./_date_nested')); - loadTestFile(require.resolve('./_search_on_page_load')); loadTestFile(require.resolve('./_chart_hidden')); loadTestFile(require.resolve('./_context_encoded_url_params')); - loadTestFile(require.resolve('./_hide_announcements')); loadTestFile(require.resolve('./_data_view_edit')); loadTestFile(require.resolve('./_field_list_new_fields')); - loadTestFile(require.resolve('./_request_cancellation')); - loadTestFile(require.resolve('./_new_search')); loadTestFile(require.resolve('./_document_comparison')); }); } diff --git a/test/functional/apps/discover/group1/_field_data.ts b/test/functional/apps/discover/group5/_field_data.ts similarity index 100% rename from test/functional/apps/discover/group1/_field_data.ts rename to test/functional/apps/discover/group5/_field_data.ts diff --git a/test/functional/apps/discover/group1/_field_data_with_fields_api.ts b/test/functional/apps/discover/group5/_field_data_with_fields_api.ts similarity index 100% rename from test/functional/apps/discover/group1/_field_data_with_fields_api.ts rename to test/functional/apps/discover/group5/_field_data_with_fields_api.ts diff --git a/test/functional/apps/discover/group1/_filter_editor.ts b/test/functional/apps/discover/group5/_filter_editor.ts similarity index 100% rename from test/functional/apps/discover/group1/_filter_editor.ts rename to test/functional/apps/discover/group5/_filter_editor.ts diff --git a/test/functional/apps/discover/group1/_greeting_screen.ts b/test/functional/apps/discover/group5/_greeting_screen.ts similarity index 100% rename from test/functional/apps/discover/group1/_greeting_screen.ts rename to test/functional/apps/discover/group5/_greeting_screen.ts diff --git a/test/functional/apps/discover/group1/_inspector.ts b/test/functional/apps/discover/group5/_inspector.ts similarity index 100% rename from test/functional/apps/discover/group1/_inspector.ts rename to test/functional/apps/discover/group5/_inspector.ts diff --git a/test/functional/apps/discover/group1/_large_string.ts b/test/functional/apps/discover/group5/_large_string.ts similarity index 100% rename from test/functional/apps/discover/group1/_large_string.ts rename to test/functional/apps/discover/group5/_large_string.ts diff --git a/test/functional/apps/discover/group1/_no_data.ts b/test/functional/apps/discover/group5/_no_data.ts similarity index 100% rename from test/functional/apps/discover/group1/_no_data.ts rename to test/functional/apps/discover/group5/_no_data.ts diff --git a/test/functional/apps/discover/group1/_shared_links.ts b/test/functional/apps/discover/group5/_shared_links.ts similarity index 100% rename from test/functional/apps/discover/group1/_shared_links.ts rename to test/functional/apps/discover/group5/_shared_links.ts diff --git a/test/functional/apps/discover/group1/_source_filters.ts b/test/functional/apps/discover/group5/_source_filters.ts similarity index 100% rename from test/functional/apps/discover/group1/_source_filters.ts rename to test/functional/apps/discover/group5/_source_filters.ts diff --git a/test/functional/apps/discover/group1/_url_state.ts b/test/functional/apps/discover/group5/_url_state.ts similarity index 100% rename from test/functional/apps/discover/group1/_url_state.ts rename to test/functional/apps/discover/group5/_url_state.ts diff --git a/test/functional/apps/discover/group5/config.ts b/test/functional/apps/discover/group5/config.ts new file mode 100644 index 00000000000000..a70a190ca63f81 --- /dev/null +++ b/test/functional/apps/discover/group5/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/test/functional/apps/discover/group5/index.ts b/test/functional/apps/discover/group5/index.ts new file mode 100644 index 00000000000000..bc875494307d60 --- /dev/null +++ b/test/functional/apps/discover/group5/index.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group5', function () { + before(async function () { + await browser.setWindowSize(1300, 800); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_no_data')); + loadTestFile(require.resolve('./_filter_editor')); + loadTestFile(require.resolve('./_field_data')); + loadTestFile(require.resolve('./_field_data_with_fields_api')); + loadTestFile(require.resolve('./_shared_links')); + loadTestFile(require.resolve('./_source_filters')); + loadTestFile(require.resolve('./_large_string')); + loadTestFile(require.resolve('./_greeting_screen')); + loadTestFile(require.resolve('./_inspector')); + loadTestFile(require.resolve('./_url_state')); + }); +} diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group6/_sidebar.ts similarity index 100% rename from test/functional/apps/discover/group3/_sidebar.ts rename to test/functional/apps/discover/group6/_sidebar.ts diff --git a/test/functional/apps/discover/group3/_sidebar_field_stats.ts b/test/functional/apps/discover/group6/_sidebar_field_stats.ts similarity index 100% rename from test/functional/apps/discover/group3/_sidebar_field_stats.ts rename to test/functional/apps/discover/group6/_sidebar_field_stats.ts diff --git a/test/functional/apps/discover/group3/_time_field_column.ts b/test/functional/apps/discover/group6/_time_field_column.ts similarity index 100% rename from test/functional/apps/discover/group3/_time_field_column.ts rename to test/functional/apps/discover/group6/_time_field_column.ts diff --git a/test/functional/apps/discover/group3/_unsaved_changes_badge.ts b/test/functional/apps/discover/group6/_unsaved_changes_badge.ts similarity index 100% rename from test/functional/apps/discover/group3/_unsaved_changes_badge.ts rename to test/functional/apps/discover/group6/_unsaved_changes_badge.ts diff --git a/test/functional/apps/discover/group3/_view_mode_toggle.ts b/test/functional/apps/discover/group6/_view_mode_toggle.ts similarity index 100% rename from test/functional/apps/discover/group3/_view_mode_toggle.ts rename to test/functional/apps/discover/group6/_view_mode_toggle.ts diff --git a/test/functional/apps/discover/group6/config.ts b/test/functional/apps/discover/group6/config.ts new file mode 100644 index 00000000000000..a70a190ca63f81 --- /dev/null +++ b/test/functional/apps/discover/group6/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/test/functional/apps/discover/group6/index.ts b/test/functional/apps/discover/group6/index.ts new file mode 100644 index 00000000000000..f71d96e63d2fd4 --- /dev/null +++ b/test/functional/apps/discover/group6/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group6', function () { + before(async function () { + await browser.setWindowSize(1300, 800); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_sidebar')); + loadTestFile(require.resolve('./_sidebar_field_stats')); + loadTestFile(require.resolve('./_time_field_column')); + loadTestFile(require.resolve('./_unsaved_changes_badge')); + loadTestFile(require.resolve('./_view_mode_toggle')); + }); +} diff --git a/test/functional/apps/discover/group4/_huge_fields.ts b/test/functional/apps/discover/group7/_huge_fields.ts similarity index 100% rename from test/functional/apps/discover/group4/_huge_fields.ts rename to test/functional/apps/discover/group7/_huge_fields.ts diff --git a/test/functional/apps/discover/group4/_indexpattern_with_unmapped_fields.ts b/test/functional/apps/discover/group7/_indexpattern_with_unmapped_fields.ts similarity index 100% rename from test/functional/apps/discover/group4/_indexpattern_with_unmapped_fields.ts rename to test/functional/apps/discover/group7/_indexpattern_with_unmapped_fields.ts diff --git a/test/functional/apps/discover/group4/_indexpattern_without_timefield.ts b/test/functional/apps/discover/group7/_indexpattern_without_timefield.ts similarity index 100% rename from test/functional/apps/discover/group4/_indexpattern_without_timefield.ts rename to test/functional/apps/discover/group7/_indexpattern_without_timefield.ts diff --git a/test/functional/apps/discover/group4/_new_search.ts b/test/functional/apps/discover/group7/_new_search.ts similarity index 100% rename from test/functional/apps/discover/group4/_new_search.ts rename to test/functional/apps/discover/group7/_new_search.ts diff --git a/test/functional/apps/discover/group4/_request_cancellation.ts b/test/functional/apps/discover/group7/_request_cancellation.ts similarity index 100% rename from test/functional/apps/discover/group4/_request_cancellation.ts rename to test/functional/apps/discover/group7/_request_cancellation.ts diff --git a/test/functional/apps/discover/group4/_runtime_fields_editor.ts b/test/functional/apps/discover/group7/_runtime_fields_editor.ts similarity index 100% rename from test/functional/apps/discover/group4/_runtime_fields_editor.ts rename to test/functional/apps/discover/group7/_runtime_fields_editor.ts diff --git a/test/functional/apps/discover/group4/_search_on_page_load.ts b/test/functional/apps/discover/group7/_search_on_page_load.ts similarity index 100% rename from test/functional/apps/discover/group4/_search_on_page_load.ts rename to test/functional/apps/discover/group7/_search_on_page_load.ts diff --git a/test/functional/apps/discover/group7/config.ts b/test/functional/apps/discover/group7/config.ts new file mode 100644 index 00000000000000..a70a190ca63f81 --- /dev/null +++ b/test/functional/apps/discover/group7/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/test/functional/apps/discover/group7/index.ts b/test/functional/apps/discover/group7/index.ts new file mode 100644 index 00000000000000..3abc84514f15de --- /dev/null +++ b/test/functional/apps/discover/group7/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group7', function () { + before(async function () { + await browser.setWindowSize(1600, 1200); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_indexpattern_without_timefield')); + loadTestFile(require.resolve('./_indexpattern_with_unmapped_fields')); + loadTestFile(require.resolve('./_runtime_fields_editor')); + loadTestFile(require.resolve('./_huge_fields')); + loadTestFile(require.resolve('./_search_on_page_load')); + loadTestFile(require.resolve('./_request_cancellation')); + loadTestFile(require.resolve('./_new_search')); + }); +} diff --git a/test/functional/apps/discover/group4/_hide_announcements.ts b/test/functional/apps/discover/group8/_hide_announcements.ts similarity index 100% rename from test/functional/apps/discover/group4/_hide_announcements.ts rename to test/functional/apps/discover/group8/_hide_announcements.ts diff --git a/test/functional/apps/discover/group8/config.ts b/test/functional/apps/discover/group8/config.ts new file mode 100644 index 00000000000000..a70a190ca63f81 --- /dev/null +++ b/test/functional/apps/discover/group8/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/test/functional/apps/discover/group8/index.ts b/test/functional/apps/discover/group8/index.ts new file mode 100644 index 00000000000000..09aaca23e8b95e --- /dev/null +++ b/test/functional/apps/discover/group8/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group8', function () { + before(async function () { + await browser.setWindowSize(1600, 1200); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_hide_announcements')); + }); +} diff --git a/test/functional/firefox/discover.config.ts b/test/functional/firefox/discover.config.ts index 8b7e7205cd4342..5c9f9c09397546 100644 --- a/test/functional/firefox/discover.config.ts +++ b/test/functional/firefox/discover.config.ts @@ -19,9 +19,15 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { testFiles: [ require.resolve('../apps/discover/classic'), require.resolve('../apps/discover/group1'), - require.resolve('../apps/discover/group2'), + require.resolve('../apps/discover/group2_data_grid1'), + require.resolve('../apps/discover/group2_data_grid2'), + require.resolve('../apps/discover/group2_data_grid3'), require.resolve('../apps/discover/group3'), require.resolve('../apps/discover/group4'), + require.resolve('../apps/discover/group5'), + require.resolve('../apps/discover/group6'), + require.resolve('../apps/discover/group7'), + require.resolve('../apps/discover/group8'), ], junit: { diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group1/index.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group1/index.ts index 4ad60320df38b0..9fc6d4447e8c74 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover/group1/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover/group1/index.ts @@ -22,6 +22,5 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_discover')); loadTestFile(require.resolve('./_discover_histogram')); - loadTestFile(require.resolve('./_url_state')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group2/index.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group2/index.ts index c579eca3bb7bd6..658b92845ffca6 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover/group2/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover/group2/index.ts @@ -22,6 +22,5 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_data_grid_doc_navigation')); loadTestFile(require.resolve('./_data_grid_doc_table')); - loadTestFile(require.resolve('./_adhoc_data_views')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group3/index.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group3/index.ts index 9f322013d986ba..70d95fef419581 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover/group3/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover/group3/index.ts @@ -20,8 +20,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); }); - loadTestFile(require.resolve('./_sidebar')); loadTestFile(require.resolve('./_request_counts')); - loadTestFile(require.resolve('./_unsaved_changes_badge')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group2/_adhoc_data_views.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group4/_adhoc_data_views.ts similarity index 100% rename from x-pack/test_serverless/functional/test_suites/common/discover/group2/_adhoc_data_views.ts rename to x-pack/test_serverless/functional/test_suites/common/discover/group4/_adhoc_data_views.ts diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group4/index.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group4/index.ts new file mode 100644 index 00000000000000..c2627980650008 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/discover/group4/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group4', function () { + before(async function () { + await browser.setWindowSize(1600, 1200); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_adhoc_data_views')); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group1/_url_state.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group5/_url_state.ts similarity index 100% rename from x-pack/test_serverless/functional/test_suites/common/discover/group1/_url_state.ts rename to x-pack/test_serverless/functional/test_suites/common/discover/group5/_url_state.ts diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group5/index.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group5/index.ts new file mode 100644 index 00000000000000..a38a6d4e33dd65 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/discover/group5/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group5', function () { + before(async function () { + await browser.setWindowSize(1300, 800); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_url_state')); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group3/_sidebar.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group6/_sidebar.ts similarity index 100% rename from x-pack/test_serverless/functional/test_suites/common/discover/group3/_sidebar.ts rename to x-pack/test_serverless/functional/test_suites/common/discover/group6/_sidebar.ts diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group3/_unsaved_changes_badge.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group6/_unsaved_changes_badge.ts similarity index 100% rename from x-pack/test_serverless/functional/test_suites/common/discover/group3/_unsaved_changes_badge.ts rename to x-pack/test_serverless/functional/test_suites/common/discover/group6/_unsaved_changes_badge.ts diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/group6/index.ts b/x-pack/test_serverless/functional/test_suites/common/discover/group6/index.ts new file mode 100644 index 00000000000000..8857ebe9bf310e --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/discover/group6/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + + describe('discover/group6', function () { + before(async function () { + await browser.setWindowSize(1300, 800); + }); + + after(async function unloadMakelogs() { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); + + loadTestFile(require.resolve('./_sidebar')); + loadTestFile(require.resolve('./_unsaved_changes_badge')); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts b/x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts index 7b8fb4b0728475..be3c0098d35d2c 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts @@ -16,6 +16,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('../../common/discover/group1'), require.resolve('../../common/discover/group2'), require.resolve('../../common/discover/group3'), + require.resolve('../../common/discover/group4'), + require.resolve('../../common/discover/group5'), + require.resolve('../../common/discover/group6'), ], junit: { reportName: 'Serverless Observability Functional Tests - Common Group 5', diff --git a/x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts b/x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts index 70cabf59051a93..ad661b474a33dd 100644 --- a/x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts +++ b/x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts @@ -16,6 +16,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('../../common/discover/group1'), require.resolve('../../common/discover/group2'), require.resolve('../../common/discover/group3'), + require.resolve('../../common/discover/group4'), + require.resolve('../../common/discover/group5'), + require.resolve('../../common/discover/group6'), ], junit: { reportName: 'Serverless Search Functional Tests - Common Group 5', diff --git a/x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts b/x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts index d1637bf34b4fd7..c65131e27e9e81 100644 --- a/x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts +++ b/x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts @@ -16,6 +16,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('../../common/discover/group1'), require.resolve('../../common/discover/group2'), require.resolve('../../common/discover/group3'), + require.resolve('../../common/discover/group4'), + require.resolve('../../common/discover/group5'), + require.resolve('../../common/discover/group6'), ], junit: { reportName: 'Serverless Security Functional Tests - Common Group 5', From 1eb43c1c797a8a37f1db32e0d611b9a83bd0fb38 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:00:16 +0100 Subject: [PATCH 026/138] skip flaky suite (#170674) --- .../cypress/e2e/response_actions/document_signing.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts index b8063237260185..4093581366321e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts @@ -22,7 +22,8 @@ import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; -describe('Document signing:', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/170674 +describe.skip('Document signing:', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; From fadce7939195ef903f116752db13ddb003eaf1a6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:01:04 -0400 Subject: [PATCH 027/138] skip failing test suite (#170674) --- .../cypress/e2e/response_actions/document_signing.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts index 4093581366321e..ec41ffa31edc9e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts @@ -23,6 +23,7 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170674 +// Failing: See https://github.com/elastic/kibana/issues/170674 describe.skip('Document signing:', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; From 14bf23cd0e871742da31ffe4f5114ef9ed9d9dda Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:02:26 +0100 Subject: [PATCH 028/138] skip flaky suite (#170811) --- ...t_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts index 3d92528c2eee72..dfa36e67bb0303 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts @@ -22,7 +22,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170811 +describe.skip( 'Unenroll agent from fleet when agent tamper protection is disabled but then is switched to a policy with it enabled', { tags: ['@ess'] }, () => { From 95f9163ddb6a194a1d6b9fe882e6abb2544e3744 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:03:06 -0400 Subject: [PATCH 029/138] skip failing test suite (#170811) --- ...ent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts index dfa36e67bb0303..03eef1c4337c27 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_disabled_to_enabled.cy.ts @@ -23,6 +23,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170811 +// Failing: See https://github.com/elastic/kibana/issues/170811 describe.skip( 'Unenroll agent from fleet when agent tamper protection is disabled but then is switched to a policy with it enabled', { tags: ['@ess'] }, From 19f6fc1d1906e7b98874f00812c58c601adba58d Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:03:37 -0400 Subject: [PATCH 030/138] skip failing test suite (#169821) --- .../e2e/response_actions/endpoints_list_response_console.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/endpoints_list_response_console.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/endpoints_list_response_console.cy.ts index 75074b0d3f94ab..10a89086845777 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/endpoints_list_response_console.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/endpoints_list_response_console.cy.ts @@ -20,7 +20,8 @@ import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; -describe('Response console', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { +// Failing: See https://github.com/elastic/kibana/issues/169821 +describe.skip('Response console', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { beforeEach(() => { login(); }); From fefe90889fa02b5e1c05682d3303d7fd40fc1da1 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:05:01 +0100 Subject: [PATCH 031/138] skip flaky suite (#170817) --- ...t_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts index a9508a13f719b3..7ffc7b39c373b4 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts @@ -22,7 +22,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170817 +describe.skip( 'Unenroll agent from fleet changing when agent tamper protection is enabled but then is switched to a policy with it disabled', { tags: ['@ess'] }, () => { From 4ba1be2b306baef0329d70710df60cff181e5fcc Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:05:35 -0400 Subject: [PATCH 032/138] skip failing test suite (#170817) --- ...ent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts index 7ffc7b39c373b4..b36f7eca756e2b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_disabled.cy.ts @@ -23,6 +23,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170817 +// Failing: See https://github.com/elastic/kibana/issues/170817 describe.skip( 'Unenroll agent from fleet changing when agent tamper protection is enabled but then is switched to a policy with it disabled', { tags: ['@ess'] }, From 5f51e05ffe561524c65314e6bd6145368be4a57c Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:06:23 +0100 Subject: [PATCH 033/138] skip flaky suite (#170816) --- ...nt_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts index a5654734c15e43..bacea254e34c30 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts @@ -21,7 +21,8 @@ import { login } from '../../../tasks/login'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170816 +describe.skip( 'Unenroll agent from fleet changing agent policy when agent tamper protection is enabled but then is switched to a policy with it also enabled', { tags: ['@ess'] }, () => { From f5877fada114a443f3e699c370ed4cb8a96f6c4a Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:06:59 -0400 Subject: [PATCH 034/138] skip failing test suite (#170816) --- ...gent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts index bacea254e34c30..414f18fb1fac72 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/unenroll_agent_from_fleet_changing_policy_from_enabled_to_enabled.cy.ts @@ -22,6 +22,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170816 +// Failing: See https://github.com/elastic/kibana/issues/170816 describe.skip( 'Unenroll agent from fleet changing agent policy when agent tamper protection is enabled but then is switched to a policy with it also enabled', { tags: ['@ess'] }, From d11a5e77f67a05dfd5ab5cfaac683b9ee457493c Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:08:17 +0100 Subject: [PATCH 035/138] skip flaky suite (#170601) --- .../tamper_protection/enabled/uninstall_agent_from_host.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts index 527566bed608b1..8f45e3d70b5e63 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts @@ -22,7 +22,8 @@ import { login } from '../../../tasks/login'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170601 +describe.skip( 'Uninstall agent from host when agent tamper protection is enabled', { tags: ['@ess'] }, () => { From 8d79cebadec8e492f252b9a786507472ac5874ee Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:08:53 -0400 Subject: [PATCH 036/138] skip failing test suite (#170601) --- .../tamper_protection/enabled/uninstall_agent_from_host.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts index 8f45e3d70b5e63..7cbcaf361a38c9 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/uninstall_agent_from_host.cy.ts @@ -23,6 +23,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170601 +// Failing: See https://github.com/elastic/kibana/issues/170601 describe.skip( 'Uninstall agent from host when agent tamper protection is enabled', { tags: ['@ess'] }, From 33c6a6b01f515be7035411171996ff1b4b896c31 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:10:51 +0100 Subject: [PATCH 037/138] skip flaky suite (#179598) --- .../cypress/e2e/response_actions/alerts_response_console.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts index 63428ec6018d2f..a9b9dc0c1f1a33 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts @@ -27,7 +27,8 @@ import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/179598 +describe.skip( 'Response console: From Alerts', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { From 13bf1b02118c0cd5b7a64650a88355dbbbf1e25b Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:11:25 -0400 Subject: [PATCH 038/138] skip failing test suite (#179598) --- .../cypress/e2e/response_actions/alerts_response_console.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts index a9b9dc0c1f1a33..db1477916f75a0 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/alerts_response_console.cy.ts @@ -28,6 +28,7 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/179598 +// Failing: See https://github.com/elastic/kibana/issues/179598 describe.skip( 'Response console: From Alerts', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, From 5b92a879577149f4b35d8701b1690e87bf4da5d8 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:12:09 +0100 Subject: [PATCH 039/138] skip flaky suite (#170563) --- .../response_actions/response_console/process_operations.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts index f7c70ebf8c7e9f..0f6da6fb9fad12 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts @@ -26,7 +26,8 @@ import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_ const AGENT_BEAT_FILE_PATH_SUFFIX = '/components/agentbeat'; -describe('Response console', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/170563 +describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); }); From 0fdc7ecaf1708a8bf2dd74ed92d26578f35f5808 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:13:25 +0100 Subject: [PATCH 040/138] skip flaky suite (#172326) --- .../e2e/response_actions/response_console/release.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts index 47d30ad96699cb..d4c72699efd321 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts @@ -27,7 +27,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe('Response console', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/172326 +describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; From 0702f99e8ba24ad57e4cb5bd2935ffa2c8a8435b Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:14:27 -0400 Subject: [PATCH 041/138] skip failing test suite (#172326) --- .../cypress/e2e/response_actions/response_console/release.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts index d4c72699efd321..50630b8fc1b46c 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts @@ -28,6 +28,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/172326 +// Failing: See https://github.com/elastic/kibana/issues/172326 describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; From c72c714ed36ff30eb65a1114c08061cc2455f639 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:16:19 +0100 Subject: [PATCH 042/138] skip flaky suite (#168340) --- .../automated_response_actions.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts index adaaf9c99059a3..30a961858b52c6 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts @@ -20,7 +20,8 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/168340 +describe.skip( 'Automated Response Actions', { tags: ['@ess', '@serverless'], From dff298da4a88d3d2b1416b38142aed417c645a47 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:16:56 -0400 Subject: [PATCH 043/138] skip failing test suite (#168340) --- .../automated_response_actions/automated_response_actions.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts index 30a961858b52c6..c295de7dd0b3ed 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts @@ -21,6 +21,7 @@ import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_dat import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; // FLAKY: https://github.com/elastic/kibana/issues/168340 +// Failing: See https://github.com/elastic/kibana/issues/168340 describe.skip( 'Automated Response Actions', { From 971c08e537f52d32e6976be965230069bdc88b9a Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:17:18 +0100 Subject: [PATCH 044/138] skip flaky suite (#170424) --- .../response_actions/response_console/file_operations.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts index 70b008f7eda7b5..38f442dec0e632 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts @@ -21,7 +21,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe('Response console', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/170424 +describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); }); From 7e4082243dffba5f958d9dc317d642c78526a97e Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:18:40 +0100 Subject: [PATCH 045/138] skip flaky suite (#169958) --- .../public/management/cypress/e2e/endpoint_alerts.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts index 06b33141bad1b8..e4f913d851735e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts @@ -19,7 +19,8 @@ import { login, ROLE } from '../tasks/login'; import { EXECUTE_ROUTE } from '../../../../common/endpoint/constants'; import { waitForActionToComplete } from '../tasks/response_actions'; -describe('Endpoint generated alerts', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/169958 +describe.skip('Endpoint generated alerts', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; From 0dae7073f256a5e562a1580fd447b6b52f48845c Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:20:01 -0400 Subject: [PATCH 046/138] skip failing test suite (#169958) --- .../public/management/cypress/e2e/endpoint_alerts.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts index e4f913d851735e..2de94b7ca05575 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_alerts.cy.ts @@ -20,6 +20,7 @@ import { EXECUTE_ROUTE } from '../../../../common/endpoint/constants'; import { waitForActionToComplete } from '../tasks/response_actions'; // FLAKY: https://github.com/elastic/kibana/issues/169958 +// Failing: See https://github.com/elastic/kibana/issues/169958 describe.skip('Endpoint generated alerts', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; From c630c7f21b9b860930f780723a5a85dd9f372ec4 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:21:48 +0100 Subject: [PATCH 047/138] skip flaky suite (#168284) --- .../management/cypress/e2e/endpoint_list/endpoints.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts index 4396937e572289..0aeb46b32fb43a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts @@ -28,7 +28,8 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; -describe('Endpoints page', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/168284 +describe.skip('Endpoints page', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; From bac14aa30bb8a590b9be3005a3018211f0d4a303 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:22:25 -0400 Subject: [PATCH 048/138] skip failing test suite (#168284) --- .../public/management/cypress/e2e/endpoint_list/endpoints.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts index 0aeb46b32fb43a..c0b12f6bd700c7 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts @@ -29,6 +29,7 @@ import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_dat import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; // FLAKY: https://github.com/elastic/kibana/issues/168284 +// Failing: See https://github.com/elastic/kibana/issues/168284 describe.skip('Endpoints page', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; From 55556e252bffbe9f69023c65bd9ce6144fb9a88c Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:22:44 +0100 Subject: [PATCH 049/138] skip flaky suite (#170812) --- ...ent_from_host_changing_policy_from_enabled_to_enabled.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts index d8630a50a83b99..f2397225a10a1c 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts @@ -23,7 +23,8 @@ import { login } from '../../../tasks/login'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170812 +describe.skip( 'Uninstall agent from host changing agent policy when agent tamper protection is enabled but then is switched to a policy with it also enabled', { tags: ['@ess'] }, () => { From 520f06320c362679298dab57d6c006563dfd48a6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:23:17 -0400 Subject: [PATCH 050/138] skip failing test suite (#170812) --- ...agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts index f2397225a10a1c..d0b16f5b757ff1 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_enabled.cy.ts @@ -24,6 +24,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170812 +// Failing: See https://github.com/elastic/kibana/issues/170812 describe.skip( 'Uninstall agent from host changing agent policy when agent tamper protection is enabled but then is switched to a policy with it also enabled', { tags: ['@ess'] }, From f77d3c9b7b1adb3e7b5f7b7576391e9d005bf96d Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:24:23 +0100 Subject: [PATCH 051/138] skip flaky suite (#170373) --- .../e2e/response_actions/response_console/execute.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts index dad573bb09c2bf..d43037f4d7f978 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts @@ -21,7 +21,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe('Response console', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/170373 +describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); }); From fb9c906d48bd7e626ee310e8fe67e9c242d581d8 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:28:08 +0100 Subject: [PATCH 052/138] skip flaky suite (#170667) --- .../tamper_protection/disabled/uninstall_agent_from_host.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts index ed47855ac894a6..34aba3fcfccf2e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts @@ -21,7 +21,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170667 +describe.skip( 'Uninstall agent from host when agent tamper protection is disabled', { tags: ['@ess'] }, () => { From b8e8b5ebfec650d145ec868328aedbb861e8bf1c Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:28:41 -0400 Subject: [PATCH 053/138] skip failing test suite (#170667) --- .../tamper_protection/disabled/uninstall_agent_from_host.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts index 34aba3fcfccf2e..a32932e0ed5080 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/uninstall_agent_from_host.cy.ts @@ -22,6 +22,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170667 +// Failing: See https://github.com/elastic/kibana/issues/170667 describe.skip( 'Uninstall agent from host when agent tamper protection is disabled', { tags: ['@ess'] }, From 908a3cd3dadf9645b9e34443be83965df1b42c47 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:29:11 +0100 Subject: [PATCH 054/138] skip flaky suite (#170604) --- ...nt_from_host_changing_policy_from_enabled_to_disabled.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts index 0768c4a49ca393..f5665d830eb4aa 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts @@ -23,7 +23,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170604 +describe.skip( 'Uninstall agent from host changing agent policy when agent tamper protection is enabled but then is switched to a policy with it disabled', { tags: ['@ess'] }, () => { From afaae2c961a457b4d60d249cc184656639c839ee Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:29:41 -0400 Subject: [PATCH 055/138] skip failing test suite (#170604) --- ...gent_from_host_changing_policy_from_enabled_to_disabled.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts index f5665d830eb4aa..665e51ea56da5a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_enabled_to_disabled.cy.ts @@ -24,6 +24,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170604 +// Failing: See https://github.com/elastic/kibana/issues/170604 describe.skip( 'Uninstall agent from host changing agent policy when agent tamper protection is enabled but then is switched to a policy with it disabled', { tags: ['@ess'] }, From b9902f6f223148859f3d28173641fcb979adf3ca Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:31:13 +0100 Subject: [PATCH 056/138] skip flaky suite (#170794) --- ...nt_from_host_changing_policy_from_disabled_to_enabled.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts index bbb675cf56d5ed..6e09cd0301f9b8 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts @@ -24,7 +24,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170794 +describe.skip( 'Uninstall agent from host changing agent policy when agent tamper protection is disabled but then is switched to a policy with it enabled', { tags: ['@ess'] }, () => { From b0962bc124494806510b2746ee81eb75f34ef0f7 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:31:45 -0400 Subject: [PATCH 057/138] skip failing test suite (#170794) --- ...gent_from_host_changing_policy_from_disabled_to_enabled.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts index 6e09cd0301f9b8..d256fb2f990b31 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/switching_policies/uninstall_agent_from_host_changing_policy_from_disabled_to_enabled.cy.ts @@ -25,6 +25,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170794 +// Failing: See https://github.com/elastic/kibana/issues/170794 describe.skip( 'Uninstall agent from host changing agent policy when agent tamper protection is disabled but then is switched to a policy with it enabled', { tags: ['@ess'] }, From 5e9042dd9d12be582f1cb6d7dfcabe9cf43ecefe Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:32:23 +0100 Subject: [PATCH 058/138] skip flaky suite (#170814) --- .../tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts index e0b26bc2f77dd7..3edf2d1327d746 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts @@ -20,7 +20,8 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170814 +describe.skip( 'Unenroll agent from fleet with agent tamper protection is disabled', { tags: ['@ess'] }, () => { From 0a8fcc9d8d91eed1e0ed5531c0f9ab7368facf23 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:33:13 -0400 Subject: [PATCH 059/138] skip failing test suite (#170814) --- .../tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts index 3edf2d1327d746..3a35f49d0ddcf8 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/disabled/unenroll_agent_from_fleet.cy.ts @@ -21,6 +21,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170814 +// Failing: See https://github.com/elastic/kibana/issues/170814 describe.skip( 'Unenroll agent from fleet with agent tamper protection is disabled', { tags: ['@ess'] }, From 7fd9faec4b0815f3dbd2a3f03d5b2f18cf852d58 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 24 Apr 2024 18:33:47 +0100 Subject: [PATCH 060/138] skip flaky suite (#170706) --- .../tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts index 17cb52c2cb0424..f2aef24ad5e12f 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts @@ -20,7 +20,8 @@ import { login } from '../../../tasks/login'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -describe( +// FLAKY: https://github.com/elastic/kibana/issues/170706 +describe.skip( 'Unenroll agent from fleet when agent tamper protection is enabled', { tags: ['@ess'] }, () => { From 2b2fe3903b8cc21c77b6304340285d47693f8a60 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:34:17 -0400 Subject: [PATCH 061/138] skip failing test suite (#170706) --- .../tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts index f2aef24ad5e12f..61af90092c06db 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/tamper_protection/enabled/unenroll_agent_from_fleet.cy.ts @@ -21,6 +21,7 @@ import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; // FLAKY: https://github.com/elastic/kibana/issues/170706 +// Failing: See https://github.com/elastic/kibana/issues/170706 describe.skip( 'Unenroll agent from fleet when agent tamper protection is enabled', { tags: ['@ess'] }, From 6d06a565b79249f81f868a9d494fea5cf6903989 Mon Sep 17 00:00:00 2001 From: Rodney Norris Date: Wed, 24 Apr 2024 13:23:10 -0500 Subject: [PATCH 062/138] [Console] Allow persistent console to be resizable (#180985) ## Summary Updates the Persistent console to be resizable by the user using an `EuiResizableButton` at the top of the console flyout. - Persistent console now defaults to the maximum size for the window - Top of console can be dragged to resize - On resize console size is saved to local storage and used as default - Double-clicking resize border will maximize console, or set it to 50% height if currently at the max height. https://github.com/elastic/kibana/assets/1972968/46c8da24-56c8-4bda-82f9-f9498ec209a0 --- .../embeddable/_embeddable_console.scss | 47 +----- .../containers/embeddable/_variables.scss | 7 - .../embeddable/console_resize_button.tsx | 142 ++++++++++++++++++ .../containers/embeddable/console_wrapper.tsx | 14 +- .../embeddable/embeddable_console.tsx | 96 ++++++++---- .../containers/embeddable/index.tsx | 7 +- src/plugins/console/public/index.ts | 1 - src/plugins/console/public/plugin.ts | 27 +++- .../services/embeddable_console.test.ts | 31 +++- .../public/services/embeddable_console.ts | 20 +++ .../public/types/embeddable_console.ts | 12 +- .../public/types/plugin_dependencies.ts | 4 +- 12 files changed, 297 insertions(+), 111 deletions(-) create mode 100644 src/plugins/console/public/application/containers/embeddable/console_resize_button.tsx diff --git a/src/plugins/console/public/application/containers/embeddable/_embeddable_console.scss b/src/plugins/console/public/application/containers/embeddable/_embeddable_console.scss index d7fa11e89f72d2..8c00712bdaadce 100644 --- a/src/plugins/console/public/application/containers/embeddable/_embeddable_console.scss +++ b/src/plugins/console/public/application/containers/embeddable/_embeddable_console.scss @@ -38,24 +38,9 @@ animation-duration: $euiAnimSpeedNormal; animation-timing-function: $euiAnimSlightResistance; animation-fill-mode: forwards; - } - - &-isOpen.embeddableConsole--large { - animation-name: embeddableConsoleOpenPanelLarge; - height: $embeddableConsoleMaxHeight; - bottom: map-get($embeddableConsoleHeights, 'l') * -1; - } - - &-isOpen.embeddableConsole--medium { - animation-name: embeddableConsoleOpenPanelMedium; - height: map-get($embeddableConsoleHeights, 'm'); - bottom: map-get($embeddableConsoleHeights, 'm') * -1; - } - - &-isOpen.embeddableConsole--small { - animation-name: embeddableConsoleOpenPanelSmall; - height: map-get($embeddableConsoleHeights, 's'); - bottom: map-get($embeddableConsoleHeights, 's') * -1; + animation-name: embeddableConsoleOpenPanel; + height: var(--embedded-console-height); + bottom: var(--embedded-console-bottom); } } @@ -80,7 +65,6 @@ &--altViewButton-container { margin-left: auto; - // padding: $euiSizeS; } } @@ -132,34 +116,13 @@ } } -@keyframes embeddableConsoleOpenPanelLarge { - 0% { - // Accounts for the initial height offset from the top - transform: translateY(calc((#{$embeddableConsoleInitialHeight} * 3) * -1)); - } - - 100% { - transform: translateY(map-get($embeddableConsoleHeights, 'l') * -1); - } -} - -@keyframes embeddableConsoleOpenPanelMedium { - 0% { - transform: translateY(-$embeddableConsoleInitialHeight); - } - - 100% { - transform: translateY(map-get($embeddableConsoleHeights, 'm') * -1); - } -} - -@keyframes embeddableConsoleOpenPanelSmall { +@keyframes embeddableConsoleOpenPanel { 0% { transform: translateY(-$embeddableConsoleInitialHeight); } 100% { - transform: translateY(map-get($embeddableConsoleHeights, 's') * -1); + transform: translateY(var(--embedded-console-bottom)); } } diff --git a/src/plugins/console/public/application/containers/embeddable/_variables.scss b/src/plugins/console/public/application/containers/embeddable/_variables.scss index 33ecd64b999c9c..9623db93b4ea79 100644 --- a/src/plugins/console/public/application/containers/embeddable/_variables.scss +++ b/src/plugins/console/public/application/containers/embeddable/_variables.scss @@ -3,10 +3,3 @@ $embeddableConsoleText: lighten(makeHighContrastColor($euiColorLightestShade, $e $embeddableConsoleBorderColor: transparentize($euiColorGhost, .8); $embeddableConsoleInitialHeight: $euiSizeXXL; $embeddableConsoleMaxHeight: calc(100vh - var(--euiFixedHeadersOffset, 0)); - -// Pixel heights ensure no blurriness caused by half pixel offsets -$embeddableConsoleHeights: ( - s: $euiSize * 30, - m: $euiSize * 50, - l: 100vh, -); diff --git a/src/plugins/console/public/application/containers/embeddable/console_resize_button.tsx b/src/plugins/console/public/application/containers/embeddable/console_resize_button.tsx new file mode 100644 index 00000000000000..0b29214594440e --- /dev/null +++ b/src/plugins/console/public/application/containers/embeddable/console_resize_button.tsx @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { useCallback, useEffect, useState, useRef } from 'react'; +import { EuiResizableButton, useEuiTheme, keys, EuiThemeComputed } from '@elastic/eui'; + +const CONSOLE_MIN_HEIGHT = 200; + +const getMouseOrTouchY = ( + e: TouchEvent | MouseEvent | React.MouseEvent | React.TouchEvent +): number => { + // Some Typescript fooling is needed here + const y = (e as TouchEvent).targetTouches + ? (e as TouchEvent).targetTouches[0].pageY + : (e as MouseEvent).pageY; + return y; +}; + +export interface EmbeddedConsoleResizeButtonProps { + consoleHeight: number; + setConsoleHeight: React.Dispatch>; +} + +export function getCurrentConsoleMaxSize(euiTheme: EuiThemeComputed<{}>) { + const euiBaseSize = parseInt(euiTheme.size.base, 10); + const winHeight = window.innerHeight; + const bodyStyle = getComputedStyle(document.body); + const headerOffset = parseInt(bodyStyle.getPropertyValue('--euiFixedHeadersOffset') ?? '0px', 10); + + // We leave a buffer of baseSize to allow room for the user to hover on the top border for resizing + return Math.max(winHeight - headerOffset - euiBaseSize, CONSOLE_MIN_HEIGHT); +} + +export const EmbeddedConsoleResizeButton = ({ + consoleHeight, + setConsoleHeight, +}: EmbeddedConsoleResizeButtonProps) => { + const { euiTheme } = useEuiTheme(); + const [maxConsoleHeight, setMaxConsoleHeight] = useState(800); + const initialConsoleHeight = useRef(consoleHeight); + const initialMouseY = useRef(0); + + useEffect(() => { + function handleResize() { + const newMaxConsoleHeight = getCurrentConsoleMaxSize(euiTheme); + // Calculate and save the console max height. This is the window height minus the header + // offset minuse the base size to allow a small buffer for grabbing the resize button. + if (maxConsoleHeight !== newMaxConsoleHeight) { + setMaxConsoleHeight(newMaxConsoleHeight); + } + if (consoleHeight > newMaxConsoleHeight && newMaxConsoleHeight > CONSOLE_MIN_HEIGHT) { + // When the current console height is greater than the new max height, + // we resize the console to the max height. This will ensure there is not weird + // behavior with the drag resize. + setConsoleHeight(newMaxConsoleHeight); + } + } + + handleResize(); + window.addEventListener('resize', handleResize); + return () => window.removeEventListener('resize', handleResize); + }, [maxConsoleHeight, euiTheme, consoleHeight, setConsoleHeight]); + const onResizeMouseMove = useCallback( + (e: MouseEvent | TouchEvent) => { + const currentMouseY = getMouseOrTouchY(e); + const mouseOffset = (currentMouseY - initialMouseY.current) * -1; + const changedConsoleHeight = initialConsoleHeight.current + mouseOffset; + + const newConsoleHeight = Math.min( + Math.max(changedConsoleHeight, CONSOLE_MIN_HEIGHT), + maxConsoleHeight + ); + + setConsoleHeight(newConsoleHeight); + }, + [maxConsoleHeight, setConsoleHeight] + ); + const onResizeMouseUp = useCallback( + (e: MouseEvent | TouchEvent) => { + initialMouseY.current = 0; + + window.removeEventListener('mousemove', onResizeMouseMove); + window.removeEventListener('mouseup', onResizeMouseUp); + window.removeEventListener('touchmove', onResizeMouseMove); + window.removeEventListener('touchend', onResizeMouseUp); + }, + [onResizeMouseMove] + ); + const onResizeMouseDown = useCallback( + (e: React.MouseEvent | React.TouchEvent) => { + initialMouseY.current = getMouseOrTouchY(e); + initialConsoleHeight.current = consoleHeight; + + // Window event listeners instead of React events are used + // in case the user's mouse leaves the component + window.addEventListener('mousemove', onResizeMouseMove); + window.addEventListener('mouseup', onResizeMouseUp); + window.addEventListener('touchmove', onResizeMouseMove); + window.addEventListener('touchend', onResizeMouseUp); + }, + [consoleHeight, onResizeMouseUp, onResizeMouseMove] + ); + const onResizeKeyDown = useCallback( + (e: React.KeyboardEvent) => { + const KEYBOARD_OFFSET = 10; + + switch (e.key) { + case keys.ARROW_UP: + e.preventDefault(); // Safari+VO will screen reader navigate off the button otherwise + setConsoleHeight((height) => Math.min(height + KEYBOARD_OFFSET, maxConsoleHeight)); + break; + case keys.ARROW_DOWN: + e.preventDefault(); // Safari+VO will screen reader navigate off the button otherwise + setConsoleHeight((height) => Math.max(height - KEYBOARD_OFFSET, CONSOLE_MIN_HEIGHT)); + } + }, + [maxConsoleHeight, setConsoleHeight] + ); + const onResizeDoubleClick = useCallback(() => { + if (consoleHeight < maxConsoleHeight) { + setConsoleHeight(maxConsoleHeight); + } else { + setConsoleHeight(maxConsoleHeight / 2); + } + }, [consoleHeight, maxConsoleHeight, setConsoleHeight]); + + return ( + + ); +}; diff --git a/src/plugins/console/public/application/containers/embeddable/console_wrapper.tsx b/src/plugins/console/public/application/containers/embeddable/console_wrapper.tsx index 6429d8894d33c3..53c75706b9da0b 100644 --- a/src/plugins/console/public/application/containers/embeddable/console_wrapper.tsx +++ b/src/plugins/console/public/application/containers/embeddable/console_wrapper.tsx @@ -29,10 +29,9 @@ import { History, Settings, Storage, - createStorage, createHistory, createSettings, - setStorage, + getStorage, } from '../../../services'; import { createUsageTracker } from '../../../services/tracker'; import { MetricsTracker, EmbeddableConsoleDependencies } from '../../../types'; @@ -78,11 +77,7 @@ const loadDependencies = async ( await loadActiveApi(core.http); const autocompleteInfo = getAutocompleteInfo(); - const storage = createStorage({ - engine: window.localStorage, - prefix: 'sense:', - }); - setStorage(storage); + const storage = getStorage(); const history = createHistory({ storage }); const settings = createSettings({ storage }); const objectStorageClient = localStorageObjectClient.create(storage); @@ -107,7 +102,10 @@ const loadDependencies = async ( }; interface ConsoleWrapperProps - extends Omit { + extends Omit< + EmbeddableConsoleDependencies, + 'setDispatch' | 'alternateView' | 'setConsoleHeight' | 'getConsoleHeight' + > { onKeyDown: (this: Window, ev: WindowEventMap['keydown']) => any; } diff --git a/src/plugins/console/public/application/containers/embeddable/embeddable_console.tsx b/src/plugins/console/public/application/containers/embeddable/embeddable_console.tsx index 218496b9d81ab5..42a6c4b0efb92f 100644 --- a/src/plugins/console/public/application/containers/embeddable/embeddable_console.tsx +++ b/src/plugins/console/public/application/containers/embeddable/embeddable_console.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import React, { useReducer, useEffect } from 'react'; +import React, { useReducer, useEffect, useState } from 'react'; import classNames from 'classnames'; import useObservable from 'react-use/lib/useObservable'; import { @@ -14,15 +14,17 @@ import { EuiFocusTrap, EuiPortal, EuiScreenReaderOnly, + EuiThemeComputed, EuiThemeProvider, EuiWindowEvent, keys, + useEuiTheme, + useEuiThemeCSSVariables, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { dynamic } from '@kbn/shared-ux-utility'; import { - EmbeddableConsoleProps, EmbeddableConsoleDependencies, EmbeddableConsoleView, } from '../../../types/embeddable_console'; @@ -31,6 +33,7 @@ import * as store from '../../stores/embeddable_console'; import { setLoadFromParameter, removeLoadFromParameter } from '../../lib/load_from'; import './_index.scss'; +import { EmbeddedConsoleResizeButton, getCurrentConsoleMaxSize } from './console_resize_button'; const KBN_BODY_CONSOLE_CLASS = 'kbnBody--hasEmbeddableConsole'; @@ -42,14 +45,39 @@ const ConsoleWrapper = dynamic(async () => ({ default: (await import('./console_wrapper')).ConsoleWrapper, })); +const getInitialConsoleHeight = ( + getConsoleHeight: EmbeddableConsoleDependencies['getConsoleHeight'], + euiTheme: EuiThemeComputed +) => { + const lastHeight = getConsoleHeight(); + if (lastHeight) { + try { + const value = parseInt(lastHeight, 10); + if (!isNaN(value) && value > 0) { + return value; + } + } catch { + // ignore bad local storage value + } + } + return getCurrentConsoleMaxSize(euiTheme); +}; + export const EmbeddableConsole = ({ - size = 'm', core, usageCollection, setDispatch, alternateView, isMonacoEnabled, -}: EmbeddableConsoleProps & EmbeddableConsoleDependencies) => { + getConsoleHeight, + setConsoleHeight, +}: EmbeddableConsoleDependencies) => { + const { euiTheme } = useEuiTheme(); + const { setGlobalCSSVariables } = useEuiThemeCSSVariables(); + const [consoleHeight, setConsoleHeightState] = useState( + getInitialConsoleHeight(getConsoleHeight, euiTheme) + ); + const [consoleState, consoleDispatch] = useReducer( store.reducer, store.initialValue, @@ -71,6 +99,13 @@ export const EmbeddableConsole = ({ document.body.classList.add(KBN_BODY_CONSOLE_CLASS); return () => document.body.classList.remove(KBN_BODY_CONSOLE_CLASS); }, []); + useEffect(() => { + setGlobalCSSVariables({ + '--embedded-console-height': `${consoleHeight}px`, + '--embedded-console-bottom': `-${consoleHeight}px`, + }); + setConsoleHeight(consoleHeight.toString()); + }, [consoleHeight, setGlobalCSSVariables, setConsoleHeight]); const isOpen = consoleState.view !== EmbeddableConsoleView.Closed; const showConsole = @@ -105,14 +140,10 @@ export const EmbeddableConsole = ({ const classes = classNames('embeddableConsole', { 'embeddableConsole-isOpen': isOpen, - 'embeddableConsole--large': size === 'l', - 'embeddableConsole--medium': size === 'm', - 'embeddableConsole--small': size === 's', 'embeddableConsole--classicChrome': chromeStyle === 'classic', 'embeddableConsole--projectChrome': chromeStyle === 'project', 'embeddableConsole--unknownChrome': chromeStyle === undefined, 'embeddableConsole--fixed': true, - 'embeddableConsole--showOnMobile': false, }); return ( @@ -127,27 +158,36 @@ export const EmbeddableConsole = ({

{landmarkHeading}

-
- - {i18n.translate('console.embeddableConsole.title', { - defaultMessage: 'Console', - })} - - {alternateView && ( -
- -
+
+ {isOpen && ( + )} + +
+ + {i18n.translate('console.embeddableConsole.title', { + defaultMessage: 'Console', + })} + + {alternateView && ( +
+ +
+ )} +
{showConsole ? ( diff --git a/src/plugins/console/public/application/containers/embeddable/index.tsx b/src/plugins/console/public/application/containers/embeddable/index.tsx index 0563a5f445da2c..0ec32dbeaac91f 100644 --- a/src/plugins/console/public/application/containers/embeddable/index.tsx +++ b/src/plugins/console/public/application/containers/embeddable/index.tsx @@ -8,12 +8,9 @@ import { dynamic } from '@kbn/shared-ux-utility'; import React from 'react'; -import { - EmbeddableConsoleProps, - EmbeddableConsoleDependencies, -} from '../../../types/embeddable_console'; +import { EmbeddableConsoleDependencies } from '../../../types/embeddable_console'; -type EmbeddableConsoleInternalProps = EmbeddableConsoleProps & EmbeddableConsoleDependencies; +type EmbeddableConsoleInternalProps = EmbeddableConsoleDependencies; const Console = dynamic(async () => ({ default: (await import('./embeddable_console')).EmbeddableConsole, })); diff --git a/src/plugins/console/public/index.ts b/src/plugins/console/public/index.ts index 4e907d4329d1ee..277190a1a443ce 100644 --- a/src/plugins/console/public/index.ts +++ b/src/plugins/console/public/index.ts @@ -17,7 +17,6 @@ export type { ConsoleUILocatorParams, ConsolePluginSetup, ConsolePluginStart, - EmbeddableConsoleProps, EmbeddedConsoleView, EmbeddedConsoleViewButtonProps, } from './types'; diff --git a/src/plugins/console/public/plugin.ts b/src/plugins/console/public/plugin.ts index 43cedf1fa4bb01..54d8d1db97bc7b 100644 --- a/src/plugins/console/public/plugin.ts +++ b/src/plugins/console/public/plugin.ts @@ -18,18 +18,30 @@ import { ConsolePluginSetup, ConsolePluginStart, ConsoleUILocatorParams, - EmbeddableConsoleProps, EmbeddedConsoleView, } from './types'; -import { AutocompleteInfo, setAutocompleteInfo, EmbeddableConsoleInfo } from './services'; +import { + AutocompleteInfo, + setAutocompleteInfo, + EmbeddableConsoleInfo, + createStorage, + setStorage, +} from './services'; export class ConsoleUIPlugin implements Plugin { private readonly autocompleteInfo = new AutocompleteInfo(); - private _embeddableConsole: EmbeddableConsoleInfo = new EmbeddableConsoleInfo(); - - constructor(private ctx: PluginInitializerContext) {} + private _embeddableConsole: EmbeddableConsoleInfo; + + constructor(private ctx: PluginInitializerContext) { + const storage = createStorage({ + engine: window.localStorage, + prefix: 'sense:', + }); + setStorage(storage); + this._embeddableConsole = new EmbeddableConsoleInfo(storage); + } public setup( { notifications, getStartServices, http }: CoreSetup, @@ -126,9 +138,8 @@ export class ConsoleUIPlugin embeddedConsoleUiSetting; if (embeddedConsoleAvailable) { - consoleStart.EmbeddableConsole = (props: EmbeddableConsoleProps) => { + consoleStart.EmbeddableConsole = (_props: {}) => { return EmbeddableConsole({ - ...props, core, usageCollection: deps.usageCollection, setDispatch: (d) => { @@ -136,6 +147,8 @@ export class ConsoleUIPlugin }, alternateView: this._embeddableConsole.alternateView, isMonacoEnabled, + getConsoleHeight: this._embeddableConsole.getConsoleHeight.bind(this._embeddableConsole), + setConsoleHeight: this._embeddableConsole.setConsoleHeight.bind(this._embeddableConsole), }); }; consoleStart.isEmbeddedConsoleAvailable = () => diff --git a/src/plugins/console/public/services/embeddable_console.test.ts b/src/plugins/console/public/services/embeddable_console.test.ts index 7df8230b6dbdf2..92cc4d84509062 100644 --- a/src/plugins/console/public/services/embeddable_console.test.ts +++ b/src/plugins/console/public/services/embeddable_console.test.ts @@ -6,12 +6,17 @@ * Side Public License, v 1. */ +import { StorageMock } from './storage.mock'; import { EmbeddableConsoleInfo } from './embeddable_console'; describe('EmbeddableConsoleInfo', () => { + jest.useFakeTimers(); + let eConsole: EmbeddableConsoleInfo; + let storage: StorageMock; beforeEach(() => { - eConsole = new EmbeddableConsoleInfo(); + storage = new StorageMock({} as unknown as Storage, 'test'); + eConsole = new EmbeddableConsoleInfo(storage); }); describe('isEmbeddedConsoleAvailable', () => { it('returns true if dispatch has been set', () => { @@ -50,4 +55,28 @@ describe('EmbeddableConsoleInfo', () => { }); }); }); + describe('getConsoleHeight', () => { + it('returns value in storage when found', () => { + storage.get.mockReturnValue('201'); + expect(eConsole.getConsoleHeight()).toEqual('201'); + expect(storage.get).toHaveBeenCalledWith('embeddedConsoleHeight', undefined); + }); + it('returns undefined when not found', () => { + storage.get.mockReturnValue(undefined); + expect(eConsole.getConsoleHeight()).toEqual(undefined); + }); + }); + describe('setConsoleHeight', () => { + it('stores value in storage', () => { + // setConsoleHeight calls are debounced + eConsole.setConsoleHeight('120'); + eConsole.setConsoleHeight('110'); + eConsole.setConsoleHeight('100'); + + jest.runAllTimers(); + + expect(storage.set).toHaveBeenCalledTimes(1); + expect(storage.set).toHaveBeenCalledWith('embeddedConsoleHeight', '100'); + }); + }); }); diff --git a/src/plugins/console/public/services/embeddable_console.ts b/src/plugins/console/public/services/embeddable_console.ts index 91bf086bc3e337..f5e0197ad833b5 100644 --- a/src/plugins/console/public/services/embeddable_console.ts +++ b/src/plugins/console/public/services/embeddable_console.ts @@ -6,16 +6,28 @@ * Side Public License, v 1. */ import type { Dispatch } from 'react'; +import { debounce } from 'lodash'; import { EmbeddedConsoleAction as EmbeddableConsoleAction, EmbeddedConsoleView, } from '../types/embeddable_console'; +import { Storage } from '.'; + +const CONSOLE_HEIGHT_KEY = 'embeddedConsoleHeight'; +const CONSOLE_HEIGHT_LOCAL_STORAGE_DEBOUNCE_WAIT_TIME = 500; export class EmbeddableConsoleInfo { private _dispatch: Dispatch | null = null; private _alternateView: EmbeddedConsoleView | undefined; + constructor(private readonly storage: Storage) { + this.setConsoleHeight = debounce( + this.setConsoleHeight.bind(this), + CONSOLE_HEIGHT_LOCAL_STORAGE_DEBOUNCE_WAIT_TIME + ); + } + public get alternateView(): EmbeddedConsoleView | undefined { return this._alternateView; } @@ -38,4 +50,12 @@ export class EmbeddableConsoleInfo { public registerAlternateView(view: EmbeddedConsoleView | null) { this._alternateView = view ?? undefined; } + + public getConsoleHeight(): string | undefined { + return this.storage.get(CONSOLE_HEIGHT_KEY, undefined); + } + + public setConsoleHeight(value: string) { + this.storage.set(CONSOLE_HEIGHT_KEY, value); + } } diff --git a/src/plugins/console/public/types/embeddable_console.ts b/src/plugins/console/public/types/embeddable_console.ts index 07a801c40287bb..9a31e0f1cf1518 100644 --- a/src/plugins/console/public/types/embeddable_console.ts +++ b/src/plugins/console/public/types/embeddable_console.ts @@ -10,22 +10,14 @@ import type { CoreStart } from '@kbn/core/public'; import type { UsageCollectionStart } from '@kbn/usage-collection-plugin/public'; import type { Dispatch } from 'react'; -/** - * EmbeddableConsoleProps are optional props used when rendering the embeddable developer console. - */ -export interface EmbeddableConsoleProps { - /** - * The default height of the content area. - */ - size?: 's' | 'm' | 'l'; -} - export interface EmbeddableConsoleDependencies { core: CoreStart; usageCollection?: UsageCollectionStart; setDispatch: (dispatch: Dispatch | null) => void; alternateView?: EmbeddedConsoleView; isMonacoEnabled: boolean; + getConsoleHeight: () => string | undefined; + setConsoleHeight: (value: string) => void; } export type EmbeddedConsoleAction = diff --git a/src/plugins/console/public/types/plugin_dependencies.ts b/src/plugins/console/public/types/plugin_dependencies.ts index 03db14c181be81..63446135e7f3c9 100644 --- a/src/plugins/console/public/types/plugin_dependencies.ts +++ b/src/plugins/console/public/types/plugin_dependencies.ts @@ -13,7 +13,7 @@ import { UsageCollectionSetup, UsageCollectionStart } from '@kbn/usage-collectio import { SharePluginSetup, SharePluginStart, LocatorPublic } from '@kbn/share-plugin/public'; import { ConsoleUILocatorParams } from './locator'; -import { EmbeddableConsoleProps, EmbeddedConsoleView } from './embeddable_console'; +import { EmbeddedConsoleView } from './embeddable_console'; export interface AppSetupUIPluginDependencies { home?: HomePublicPluginSetup; @@ -55,7 +55,7 @@ export interface ConsolePluginStart { /** * EmbeddableConsole is a functional component used to render a portable version of the dev tools console on any page in Kibana */ - EmbeddableConsole?: FC; + EmbeddableConsole?: FC<{}>; /** * Register an alternate view for the Embedded Console * From 7d13fbadea35072d07f6a4ca39b2460ee90d1a3a Mon Sep 17 00:00:00 2001 From: Saarika Bhasi <55930906+saarikabhasi@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:26:42 -0400 Subject: [PATCH 063/138] [Serverless Search] add readOnly and writeOnly privileges button in create api key flyout (#181472) ## Summary Add ready only and write only button to show privileges in code editor in`serverless_search` plugin in API key section https://github.com/elastic/kibana/assets/55930906/1e831194-218b-471a-9fd7-7737755e2c85 ### Checklist - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../api_key/security_privileges_form.tsx | 77 ++++++++++++++++++- .../page_objects/svl_search_landing_page.ts | 2 + 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/serverless_search/public/application/components/api_key/security_privileges_form.tsx b/x-pack/plugins/serverless_search/public/application/components/api_key/security_privileges_form.tsx index dcc301469837af..c647471f90e71d 100644 --- a/x-pack/plugins/serverless_search/public/application/components/api_key/security_privileges_form.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/api_key/security_privileges_form.tsx @@ -5,22 +5,51 @@ * 2.0. */ -import { EuiText, EuiLink, EuiSpacer } from '@elastic/eui'; +import { + EuiText, + EuiLink, + EuiSpacer, + EuiPanel, + EuiFlexItem, + EuiFlexGroup, + EuiButtonEmpty, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { CodeEditorField } from '@kbn/code-editor'; import React from 'react'; import { docLinks } from '../../../../common/doc_links'; - +const READ_ONLY_BOILERPLATE = `{ + "read-only-role": { + "cluster": [], + "indices": [ + { + "names": ["*"], + "privileges": ["read"] + } + ] + } +}`; +const WRITE_ONLY_BOILERPLATE = `{ + "write-only-role": { + "cluster": [], + "indices": [ + { + "names": ["*"], + "privileges": ["write"] + } + ] + } +}`; interface SecurityPrivilegesFormProps { - roleDescriptors: string; onChangeRoleDescriptors: (roleDescriptors: string) => void; error?: React.ReactNode | React.ReactNode[]; + roleDescriptors: string; } export const SecurityPrivilegesForm: React.FC = ({ - roleDescriptors, onChangeRoleDescriptors, error, + roleDescriptors, }) => { return (
@@ -39,6 +68,46 @@ export const SecurityPrivilegesForm: React.FC = ({

{error}

)} + + + + +

+ {i18n.translate('xpack.serverlessSearch.apiKey.privileges.boilerplate.label', { + defaultMessage: 'Replace with boilerplate:', + })} +

+ + + + + onChangeRoleDescriptors(READ_ONLY_BOILERPLATE)} + > + {i18n.translate( + 'xpack.serverlessSearch.apiKey.privileges.boilerplate.readOnlyLabel', + { + defaultMessage: 'Read-only', + } + )} + + + + onChangeRoleDescriptors(WRITE_ONLY_BOILERPLATE)} + > + {i18n.translate( + 'xpack.serverlessSearch.apiKey.privileges.boilerplate.writeOnlyLabel', + { + defaultMessage: 'Write-only', + } + )} + + + + Date: Wed, 24 Apr 2024 15:33:29 -0400 Subject: [PATCH 064/138] [APM] add filters support to apm latency, throughput, and error rate chart apis (#181359) ## Summary Rational: We'd like to embed APM visualizations across the observability solution, particularly within the SLO alert details page at this time. SLO configuration supports unified search filters. In order to ensure that the data accurately reflects the SLO configuration, API dependencies for APM visualizations must support filters. This PR adds filters support to: 1. `GET /internal/apm/services/{serviceName}/transactions/charts/latency` 2. `GET /internal/apm/services/{serviceName}/throughput` 3. `GET /internal/apm/services/{serviceName}/transactions/charts/error_rate` It is expected that consumers of the filters param send a serialized object containing a `filter` or `must_not` clause to include on the respective ES queries. Internally, it is expected that these objects are created using the `buildQueryFromFilters` helper exposed by `kbn/es-query`, passing the `Filter` object from the unified search `SearchBar` as the the parameter. ### Testing This feature is not yet available in the UI To test, I've added api integration tests for each api, as well as jest tests for any helpers introduced. --- .../get_failed_transaction_rate.ts | 8 +- .../server/routes/default_api_types.test.ts | 42 +++ .../apm/server/routes/default_api_types.ts | 30 +++ .../server/routes/services/get_throughput.ts | 6 +- .../apm/server/routes/services/route.ts | 5 +- .../get_failed_transaction_rate_periods.ts | 4 + .../transactions/get_latency_charts/index.ts | 12 +- .../apm/server/routes/transactions/route.ts | 10 +- .../tests/services/throughput.spec.ts | 245 ++++++++++++++++++ .../tests/transactions/error_rate.spec.ts | 132 ++++++++++ .../tests/transactions/latency.spec.ts | 120 +++++++++ 11 files changed, 606 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm/server/routes/default_api_types.test.ts diff --git a/x-pack/plugins/observability_solution/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts b/x-pack/plugins/observability_solution/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts index 08448ef50dc4b1..4b1ee98c48cd3f 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { BoolQuery } from '@kbn/es-query'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ApmServiceTransactionDocumentType } from '../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE } from '../../../common/es_fields/apm'; @@ -22,6 +22,7 @@ import { export async function getFailedTransactionRate({ environment, kuery, + filters, serviceName, transactionTypes, transactionName, @@ -35,6 +36,7 @@ export async function getFailedTransactionRate({ }: { environment: string; kuery: string; + filters?: BoolQuery; serviceName: string; transactionTypes: string[]; transactionName?: string; @@ -62,7 +64,9 @@ export async function getFailedTransactionRate({ ...rangeQuery(startWithOffset, endWithOffset), ...environmentQuery(environment), ...kqlQuery(kuery), + ...(filters?.filter || []), ]; + const mustNot = filters?.must_not || []; const outcomes = getOutcomeAggregation(documentType); @@ -73,7 +77,7 @@ export async function getFailedTransactionRate({ body: { track_total_hits: false, size: 0, - query: { bool: { filter } }, + query: { bool: { filter, must_not: mustNot } }, aggs: { ...outcomes, timeseries: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.test.ts new file mode 100644 index 00000000000000..baeda52f7fc3c5 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isLeft } from 'fp-ts/lib/Either'; +import { filtersRt } from './default_api_types'; + +describe('filtersRt', () => { + it('should decode', () => { + const filters = + '{"must_not":[{"term":{"service.name":"myService"}}],"filter":[{"range":{"@timestamp":{"gte":1617273600000,"lte":1617277200000}}}]}'; + const result = filtersRt.decode(filters); + expect(result).toEqual({ + _tag: 'Right', + right: { + should: [], + must: [], + must_not: [{ term: { 'service.name': 'myService' } }], + filter: [{ range: { '@timestamp': { gte: 1617273600000, lte: 1617277200000 } } }], + }, + }); + }); + + it.each(['3', 'true', '{}'])('should not decode invalid filter JSON: %s', (invalidJson) => { + const filters = `{ "filter": ${invalidJson}}`; + const result = filtersRt.decode(filters); + // @ts-ignore-next-line + expect(result.left[0].message).toEqual('filters.filter is not iterable'); + expect(isLeft(result)).toEqual(true); + }); + + it.each(['3', 'true', '{}'])('should not decode invalid must_not JSON: %s', (invalidJson) => { + const filters = `{ "must_not": ${invalidJson}}`; + const result = filtersRt.decode(filters); + // @ts-ignore-next-line + expect(result.left[0].message).toEqual('filters.must_not is not iterable'); + expect(isLeft(result)).toEqual(true); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.ts b/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.ts index a58a5a24af7b59..42ab1b63d431e3 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/default_api_types.ts @@ -7,6 +7,8 @@ import * as t from 'io-ts'; import { isoToEpochRt, toNumberRt } from '@kbn/io-ts-utils'; +import { either } from 'fp-ts/lib/Either'; +import { BoolQuery } from '@kbn/es-query'; import { ApmDocumentType } from '../../common/document_type'; import { RollupInterval } from '../../common/rollup'; @@ -48,3 +50,31 @@ export const transactionDataSourceRt = t.type({ t.literal(RollupInterval.None), ]), }); + +const BoolQueryRt = t.type({ + should: t.array(t.record(t.string, t.unknown)), + must: t.array(t.record(t.string, t.unknown)), + must_not: t.array(t.record(t.string, t.unknown)), + filter: t.array(t.record(t.string, t.unknown)), +}); + +export const filtersRt = new t.Type( + 'BoolQuery', + BoolQueryRt.is, + (input: unknown, context: t.Context) => + either.chain(t.string.validate(input, context), (value: string) => { + try { + const filters = JSON.parse(value); + const decoded = { + should: [], + must: [], + must_not: filters.must_not ? [...filters.must_not] : [], + filter: filters.filter ? [...filters.filter] : [], + }; + return t.success(decoded); + } catch (err) { + return t.failure(input, context, err.message); + } + }), + (filters: BoolQuery): string => JSON.stringify(filters) +); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_throughput.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_throughput.ts index 5d45ea28c95e56..b5c48484e1039e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_throughput.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_throughput.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { BoolQuery } from '@kbn/es-query'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ApmServiceTransactionDocumentType } from '../../../common/document_type'; import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE } from '../../../common/es_fields/apm'; @@ -17,6 +17,7 @@ import { Maybe } from '../../../typings/common'; interface Options { environment: string; kuery: string; + filters?: BoolQuery; serviceName: string; apmEventClient: APMEventClient; transactionType: string; @@ -34,6 +35,7 @@ export type ServiceThroughputResponse = Array<{ x: number; y: Maybe }>; export async function getThroughput({ environment, kuery, + filters, serviceName, apmEventClient, transactionType, @@ -67,7 +69,9 @@ export async function getThroughput({ ...environmentQuery(environment), ...kqlQuery(kuery), ...termQuery(TRANSACTION_NAME, transactionName), + ...(filters?.filter ?? []), ], + must_not: [...(filters?.must_not ?? [])], }, }, aggs: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts index 05fdeec8fdf5d8..4b0ef92450f34b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/route.ts @@ -33,6 +33,7 @@ import { withApmSpan } from '../../utils/with_apm_span'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, + filtersRt, kueryRt, probabilityRt, rangeRt, @@ -495,7 +496,7 @@ const serviceThroughputRoute = createApmServerRoute({ }), query: t.intersection([ t.type({ transactionType: t.string, bucketSizeInSeconds: toNumberRt }), - t.partial({ transactionName: t.string }), + t.partial({ transactionName: t.string, filters: filtersRt }), t.intersection([environmentRt, kueryRt, rangeRt, offsetRt, serviceTransactionDataSourceRt]), ]), }), @@ -512,6 +513,7 @@ const serviceThroughputRoute = createApmServerRoute({ const { environment, kuery, + filters, transactionType, transactionName, offset, @@ -525,6 +527,7 @@ const serviceThroughputRoute = createApmServerRoute({ const commonProps = { environment, kuery, + filters, serviceName, apmEventClient, transactionType, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_failed_transaction_rate_periods.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_failed_transaction_rate_periods.ts index 5b77a780bce6a2..c2ba9d1014a67a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_failed_transaction_rate_periods.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_failed_transaction_rate_periods.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { BoolQuery } from '@kbn/es-query'; import { getFailedTransactionRate } from '../../lib/transaction_groups/get_failed_transaction_rate'; import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -25,6 +26,7 @@ export interface FailedTransactionRateResponse { export async function getFailedTransactionRatePeriods({ environment, kuery, + filters, serviceName, transactionType, transactionName, @@ -38,6 +40,7 @@ export async function getFailedTransactionRatePeriods({ }: { environment: string; kuery: string; + filters?: BoolQuery; serviceName: string; transactionType: string; transactionName?: string; @@ -52,6 +55,7 @@ export async function getFailedTransactionRatePeriods({ const commonProps = { environment, kuery, + filters, serviceName, transactionTypes: [transactionType], transactionName, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_latency_charts/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_latency_charts/index.ts index f05682ca047bb3..70e9555af48495 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_latency_charts/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_latency_charts/index.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { BoolQuery } from '@kbn/es-query'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ApmServiceTransactionDocumentType } from '../../../../common/document_type'; import { @@ -29,6 +29,7 @@ import { getDurationFieldForTransactions } from '../../../lib/helpers/transactio function searchLatency({ environment, kuery, + filters, serviceName, transactionType, transactionName, @@ -45,6 +46,7 @@ function searchLatency({ }: { environment: string; kuery: string; + filters?: BoolQuery; serviceName: string; transactionType: string | undefined; transactionName: string | undefined; @@ -87,7 +89,9 @@ function searchLatency({ ...termQuery(TRANSACTION_NAME, transactionName), ...termQuery(TRANSACTION_TYPE, transactionType), ...termQuery(FAAS_ID, serverlessId), + ...(filters?.filter || []), ], + must_not: filters?.must_not || [], }, }, aggs: { @@ -111,6 +115,7 @@ function searchLatency({ export async function getLatencyTimeseries({ environment, kuery, + filters, serviceName, transactionType, transactionName, @@ -127,6 +132,7 @@ export async function getLatencyTimeseries({ }: { environment: string; kuery: string; + filters?: BoolQuery; serviceName: string; transactionType?: string; transactionName?: string; @@ -144,6 +150,7 @@ export async function getLatencyTimeseries({ const response = await searchLatency({ environment, kuery, + filters, serviceName, transactionType, transactionName, @@ -195,6 +202,7 @@ export async function getLatencyPeriods({ apmEventClient, latencyAggregationType, kuery, + filters, environment, start, end, @@ -210,6 +218,7 @@ export async function getLatencyPeriods({ apmEventClient: APMEventClient; latencyAggregationType: LatencyAggregationType; kuery: string; + filters?: BoolQuery; environment: string; start: number; end: number; @@ -225,6 +234,7 @@ export async function getLatencyPeriods({ transactionName, apmEventClient, kuery, + filters, environment, documentType, rollupInterval, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/route.ts index 8e6b8a654a030b..816879d7cb40a8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/route.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { jsonRt, toBooleanRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { offsetRt } from '../../../common/comparison_rt'; @@ -23,6 +22,7 @@ import { import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, + filtersRt, kueryRt, rangeRt, serviceTransactionDataSourceRt, @@ -221,7 +221,7 @@ const transactionLatencyChartsRoute = createApmServerRoute({ bucketSizeInSeconds: toNumberRt, useDurationSummary: toBooleanRt, }), - t.partial({ transactionName: t.string }), + t.partial({ transactionName: t.string, filters: filtersRt }), t.intersection([environmentRt, kueryRt, rangeRt, offsetRt]), serviceTransactionDataSourceRt, ]), @@ -235,6 +235,7 @@ const transactionLatencyChartsRoute = createApmServerRoute({ const { environment, kuery, + filters, transactionType, transactionName, latencyAggregationType, @@ -250,6 +251,7 @@ const transactionLatencyChartsRoute = createApmServerRoute({ const options = { environment, kuery, + filters, serviceName, transactionType, transactionName, @@ -372,7 +374,7 @@ const transactionChartsErrorRateRoute = createApmServerRoute({ }), query: t.intersection([ t.type({ transactionType: t.string, bucketSizeInSeconds: toNumberRt }), - t.partial({ transactionName: t.string }), + t.partial({ transactionName: t.string, filters: filtersRt }), t.intersection([environmentRt, kueryRt, rangeRt, offsetRt, serviceTransactionDataSourceRt]), ]), }), @@ -385,6 +387,7 @@ const transactionChartsErrorRateRoute = createApmServerRoute({ const { environment, kuery, + filters, transactionType, transactionName, start, @@ -398,6 +401,7 @@ const transactionChartsErrorRateRoute = createApmServerRoute({ return getFailedTransactionRatePeriods({ environment, kuery, + filters, serviceName, transactionType, transactionName, diff --git a/x-pack/test/apm_api_integration/tests/services/throughput.spec.ts b/x-pack/test/apm_api_integration/tests/services/throughput.spec.ts index ef56e61bf4f823..624706d30115ae 100644 --- a/x-pack/test/apm_api_integration/tests/services/throughput.spec.ts +++ b/x-pack/test/apm_api_integration/tests/services/throughput.spec.ts @@ -7,6 +7,7 @@ import { apm, timerange } from '@kbn/apm-synthtrace-client'; import expect from '@kbn/expect'; +import { buildQueryFromFilters } from '@kbn/es-query'; import { first, last, meanBy } from 'lodash'; import moment from 'moment'; import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number'; @@ -285,6 +286,250 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); }); }); + + describe('handles kuery', () => { + let throughputMetrics: ThroughputReturn; + let throughputTransactions: ThroughputReturn; + + before(async () => { + const [throughputMetricsResponse, throughputTransactionsResponse] = await Promise.all([ + callApi( + { + query: { + kuery: 'transaction.name : "GET /api/product/list"', + }, + }, + 'metric' + ), + callApi( + { + query: { + kuery: 'transaction.name : "GET /api/product/list"', + }, + }, + 'transaction' + ), + ]); + throughputMetrics = throughputMetricsResponse.body; + throughputTransactions = throughputTransactionsResponse.body; + }); + + it('returns some transactions data', () => { + expect(throughputTransactions.currentPeriod.length).to.be.greaterThan(0); + const hasData = throughputTransactions.currentPeriod.some(({ y }) => isFiniteNumber(y)); + expect(hasData).to.equal(true); + }); + + it('returns some metrics data', () => { + expect(throughputMetrics.currentPeriod.length).to.be.greaterThan(0); + const hasData = throughputMetrics.currentPeriod.some(({ y }) => isFiniteNumber(y)); + expect(hasData).to.equal(true); + }); + + it('has same mean value for metrics and transactions data', () => { + const transactionsMean = meanBy(throughputTransactions.currentPeriod, 'y'); + const metricsMean = meanBy(throughputMetrics.currentPeriod, 'y'); + [transactionsMean, metricsMean].forEach((value) => + expect(roundNumber(value)).to.be.equal(roundNumber(GO_PROD_RATE)) + ); + }); + + it('has a bucket size of 30 seconds for transactions data', () => { + const firstTimerange = throughputTransactions.currentPeriod[0].x; + const secondTimerange = throughputTransactions.currentPeriod[1].x; + const timeIntervalAsSeconds = (secondTimerange - firstTimerange) / 1000; + expect(timeIntervalAsSeconds).to.equal(30); + }); + + it('has a bucket size of 1 minute for metrics data', () => { + const firstTimerange = throughputMetrics.currentPeriod[0].x; + const secondTimerange = throughputMetrics.currentPeriod[1].x; + const timeIntervalAsMinutes = (secondTimerange - firstTimerange) / 1000 / 60; + expect(timeIntervalAsMinutes).to.equal(1); + }); + }); + + describe('handles filters', () => { + let throughputMetrics: ThroughputReturn; + let throughputTransactions: ThroughputReturn; + const filters = [ + { + meta: { + disabled: false, + negate: false, + alias: null, + key: 'transaction.name', + params: ['GET /api/product/list'], + type: 'phrases', + }, + query: { + bool: { + minimum_should_match: 1, + should: { + match_phrase: { + 'transaction.name': 'GET /api/product/list', + }, + }, + }, + }, + }, + ]; + const serializedFilters = JSON.stringify(buildQueryFromFilters(filters, undefined)); + + before(async () => { + const [throughputMetricsResponse, throughputTransactionsResponse] = await Promise.all([ + callApi( + { + query: { + filters: serializedFilters, + }, + }, + 'metric' + ), + callApi( + { + query: { + filters: serializedFilters, + }, + }, + 'transaction' + ), + ]); + throughputMetrics = throughputMetricsResponse.body; + throughputTransactions = throughputTransactionsResponse.body; + }); + + it('returns some transactions data', () => { + expect(throughputTransactions.currentPeriod.length).to.be.greaterThan(0); + const hasData = throughputTransactions.currentPeriod.some(({ y }) => isFiniteNumber(y)); + expect(hasData).to.equal(true); + }); + + it('returns some metrics data', () => { + expect(throughputMetrics.currentPeriod.length).to.be.greaterThan(0); + const hasData = throughputMetrics.currentPeriod.some(({ y }) => isFiniteNumber(y)); + expect(hasData).to.equal(true); + }); + + it('has same mean value for metrics and transactions data', () => { + const transactionsMean = meanBy(throughputTransactions.currentPeriod, 'y'); + const metricsMean = meanBy(throughputMetrics.currentPeriod, 'y'); + [transactionsMean, metricsMean].forEach((value) => + expect(roundNumber(value)).to.be.equal(roundNumber(GO_PROD_RATE)) + ); + }); + + it('has a bucket size of 30 seconds for transactions data', () => { + const firstTimerange = throughputTransactions.currentPeriod[0].x; + const secondTimerange = throughputTransactions.currentPeriod[1].x; + const timeIntervalAsSeconds = (secondTimerange - firstTimerange) / 1000; + expect(timeIntervalAsSeconds).to.equal(30); + }); + + it('has a bucket size of 1 minute for metrics data', () => { + const firstTimerange = throughputMetrics.currentPeriod[0].x; + const secondTimerange = throughputMetrics.currentPeriod[1].x; + const timeIntervalAsMinutes = (secondTimerange - firstTimerange) / 1000 / 60; + expect(timeIntervalAsMinutes).to.equal(1); + }); + }); + + describe('handles negate filters', () => { + let throughputMetrics: ThroughputReturn; + let throughputTransactions: ThroughputReturn; + const filters = [ + { + meta: { + disabled: false, + negate: true, + alias: null, + key: 'transaction.name', + params: ['GET /api/product/list'], + type: 'phrases', + }, + query: { + bool: { + minimum_should_match: 1, + should: { + match_phrase: { + 'transaction.name': 'GET /api/product/list', + }, + }, + }, + }, + }, + ]; + const serializedFilters = JSON.stringify(buildQueryFromFilters(filters, undefined)); + + before(async () => { + const [throughputMetricsResponse, throughputTransactionsResponse] = await Promise.all([ + callApi( + { + query: { + filters: serializedFilters, + }, + }, + 'metric' + ), + callApi( + { + query: { + filters: serializedFilters, + }, + }, + 'transaction' + ), + ]); + throughputMetrics = throughputMetricsResponse.body; + throughputTransactions = throughputTransactionsResponse.body; + }); + + it('returns some transactions data', () => { + expect(throughputTransactions.currentPeriod.length).to.be.greaterThan(0); + const hasData = throughputTransactions.currentPeriod.some(({ y }) => isFiniteNumber(y)); + expect(hasData).to.equal(true); + }); + + it('returns some metrics data', () => { + expect(throughputMetrics.currentPeriod.length).to.be.greaterThan(0); + const hasData = throughputMetrics.currentPeriod.some(({ y }) => isFiniteNumber(y)); + expect(hasData).to.equal(true); + }); + + it('has same mean value for metrics and transactions data', () => { + const transactionsMean = meanBy(throughputTransactions.currentPeriod, 'y'); + const metricsMean = meanBy(throughputMetrics.currentPeriod, 'y'); + [transactionsMean, metricsMean].forEach((value) => + expect(roundNumber(value)).to.be.equal(roundNumber(GO_DEV_RATE)) + ); + }); + + it('has a bucket size of 30 seconds for transactions data', () => { + const firstTimerange = throughputTransactions.currentPeriod[0].x; + const secondTimerange = throughputTransactions.currentPeriod[1].x; + const timeIntervalAsSeconds = (secondTimerange - firstTimerange) / 1000; + expect(timeIntervalAsSeconds).to.equal(30); + }); + + it('has a bucket size of 1 minute for metrics data', () => { + const firstTimerange = throughputMetrics.currentPeriod[0].x; + const secondTimerange = throughputMetrics.currentPeriod[1].x; + const timeIntervalAsMinutes = (secondTimerange - firstTimerange) / 1000 / 60; + expect(timeIntervalAsMinutes).to.equal(1); + }); + }); + + describe('handles bad filters request', () => { + it('throws bad request error', async () => { + try { + await callApi({ + query: { environment: 'production', filters: '{}}' }, + }); + } catch (error) { + expect(error.res.status).to.be(400); + } + }); + }); }); }); } diff --git a/x-pack/test/apm_api_integration/tests/transactions/error_rate.spec.ts b/x-pack/test/apm_api_integration/tests/transactions/error_rate.spec.ts index 996127103b090c..123bb0d6d594d8 100644 --- a/x-pack/test/apm_api_integration/tests/transactions/error_rate.spec.ts +++ b/x-pack/test/apm_api_integration/tests/transactions/error_rate.spec.ts @@ -6,6 +6,7 @@ */ import { apm, timerange } from '@kbn/apm-synthtrace-client'; import expect from '@kbn/expect'; +import { buildQueryFromFilters } from '@kbn/es-query'; import { first, last } from 'lodash'; import moment from 'moment'; import { @@ -297,5 +298,136 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); }); + + describe('handles kuery', () => { + let txMetricsErrorRateResponse: ErrorRate; + + before(async () => { + const txMetricsResponse = await fetchErrorCharts({ + query: { + kuery: 'transaction.name : "GET /pear 🍎 "', + }, + }); + txMetricsErrorRateResponse = txMetricsResponse.body; + }); + + describe('has the correct calculation for average with kuery', () => { + const expectedFailureRate = config.secondTransaction.failureRate / 100; + + it('for tx metrics', () => { + expect(txMetricsErrorRateResponse.currentPeriod.average).to.eql(expectedFailureRate); + }); + }); + }); + + describe('handles filters', () => { + const filters = [ + { + meta: { + disabled: false, + negate: false, + alias: null, + key: 'transaction.name', + params: ['GET /api/product/list'], + type: 'phrases', + }, + query: { + bool: { + minimum_should_match: 1, + should: { + match_phrase: { + 'transaction.name': 'GET /pear 🍎 ', + }, + }, + }, + }, + }, + ]; + const serializedFilters = JSON.stringify(buildQueryFromFilters(filters, undefined)); + let txMetricsErrorRateResponse: ErrorRate; + + before(async () => { + const txMetricsResponse = await fetchErrorCharts({ + query: { + filters: serializedFilters, + }, + }); + txMetricsErrorRateResponse = txMetricsResponse.body; + }); + + describe('has the correct calculation for average with filter', () => { + const expectedFailureRate = config.secondTransaction.failureRate / 100; + + it('for tx metrics', () => { + expect(txMetricsErrorRateResponse.currentPeriod.average).to.eql(expectedFailureRate); + }); + }); + + describe('has the correct calculation for average with negate filter', () => { + const expectedFailureRate = config.secondTransaction.failureRate / 100; + + it('for tx metrics', () => { + expect(txMetricsErrorRateResponse.currentPeriod.average).to.eql(expectedFailureRate); + }); + }); + }); + + describe('handles negate filters', () => { + const filters = [ + { + meta: { + disabled: false, + negate: true, + alias: null, + key: 'transaction.name', + params: ['GET /api/product/list'], + type: 'phrases', + }, + query: { + bool: { + minimum_should_match: 1, + should: { + match_phrase: { + 'transaction.name': 'GET /pear 🍎 ', + }, + }, + }, + }, + }, + ]; + const serializedFilters = JSON.stringify(buildQueryFromFilters(filters, undefined)); + let txMetricsErrorRateResponse: ErrorRate; + + before(async () => { + const txMetricsResponse = await fetchErrorCharts({ + query: { + filters: serializedFilters, + }, + }); + txMetricsErrorRateResponse = txMetricsResponse.body; + }); + + describe('has the correct calculation for average with filter', () => { + const expectedFailureRate = config.firstTransaction.failureRate / 100; + + it('for tx metrics', () => { + expect(txMetricsErrorRateResponse.currentPeriod.average).to.eql(expectedFailureRate); + }); + }); + }); + + describe('handles bad filters request', () => { + it('for tx metrics', async () => { + try { + await fetchErrorCharts({ + query: { + filters: '{}}}', + }, + }); + } catch (e) { + expect(e.res.status).to.eql(400); + } + }); + }); }); } diff --git a/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts b/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts index a1cea01f408ca3..eb876e6e312b76 100644 --- a/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts +++ b/x-pack/test/apm_api_integration/tests/transactions/latency.spec.ts @@ -6,6 +6,7 @@ */ import { apm, timerange } from '@kbn/apm-synthtrace-client'; import expect from '@kbn/expect'; +import { buildQueryFromFilters } from '@kbn/es-query'; import moment from 'moment'; import { APIClientRequestParamsOf, @@ -115,6 +116,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { ((GO_PROD_RATE * GO_PROD_DURATION + GO_DEV_RATE * GO_DEV_DURATION) / (GO_PROD_RATE + GO_DEV_RATE)) * 1000; + const expectedLatencyAvgValueProdMs = + ((GO_PROD_RATE * GO_PROD_DURATION) / GO_PROD_RATE) * 1000; + const expectedLatencyAvgValueDevMs = ((GO_DEV_RATE * GO_DEV_DURATION) / GO_DEV_RATE) * 1000; describe('average latency type', () => { it('returns average duration and timeseries', async () => { @@ -319,6 +323,122 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); }); + + describe('handles kuery', () => { + it('should return the appropriate latency values when a kuery is applied', async () => { + const response = await fetchLatencyCharts({ + query: { + latencyAggregationType: LatencyAggregationType.p95, + useDurationSummary: false, + kuery: 'transaction.name : "GET /api/product/list"', + }, + }); + + expect(response.status).to.be(200); + const latencyChartReturn = response.body as LatencyChartReturnType; + + expect(latencyChartReturn.currentPeriod.overallAvgDuration).to.be( + expectedLatencyAvgValueProdMs + ); + expect(latencyChartReturn.currentPeriod.latencyTimeseries.length).to.be.eql(15); + }); + }); + + describe('handles filters', () => { + it('should return the appropriate latency values when filters are applied', async () => { + const filters = [ + { + meta: { + disabled: false, + negate: false, + alias: null, + key: 'transaction.name', + params: ['GET /api/product/list'], + type: 'phrases', + }, + query: { + bool: { + minimum_should_match: 1, + should: { + match_phrase: { + 'transaction.name': 'GET /api/product/list', + }, + }, + }, + }, + }, + ]; + const serializedFilters = JSON.stringify(buildQueryFromFilters(filters, undefined)); + const response = await fetchLatencyCharts({ + query: { + latencyAggregationType: LatencyAggregationType.p95, + useDurationSummary: false, + filters: serializedFilters, + }, + }); + + expect(response.status).to.be(200); + const latencyChartReturn = response.body as LatencyChartReturnType; + + expect(latencyChartReturn.currentPeriod.overallAvgDuration).to.be( + expectedLatencyAvgValueProdMs + ); + expect(latencyChartReturn.currentPeriod.latencyTimeseries.length).to.be.eql(15); + }); + + it('should return the appropriate latency values when negate filters are applied', async () => { + const filters = [ + { + meta: { + disabled: false, + negate: true, + alias: null, + key: 'transaction.name', + params: ['GET /api/product/list'], + type: 'phrases', + }, + query: { + bool: { + minimum_should_match: 1, + should: { + match_phrase: { + 'transaction.name': 'GET /api/product/list', + }, + }, + }, + }, + }, + ]; + const serializedFilters = JSON.stringify(buildQueryFromFilters(filters, undefined)); + const response = await fetchLatencyCharts({ + query: { + latencyAggregationType: LatencyAggregationType.p95, + useDurationSummary: false, + filters: serializedFilters, + }, + }); + + expect(response.status).to.be(200); + const latencyChartReturn = response.body as LatencyChartReturnType; + + expect(latencyChartReturn.currentPeriod.overallAvgDuration).to.be( + expectedLatencyAvgValueDevMs + ); + expect(latencyChartReturn.currentPeriod.latencyTimeseries.length).to.be.eql(15); + }); + }); + + describe('handles bad filters request', () => { + it('throws bad request error', async () => { + try { + await fetchLatencyCharts({ + query: { environment: 'production', filters: '{}}' }, + }); + } catch (error) { + expect(error.res.status).to.be(400); + } + }); + }); } ); } From f52db83d33735f2ce0463d510e75b81f05dab6f1 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 24 Apr 2024 14:40:47 -0500 Subject: [PATCH 065/138] Revert "Enable heap snapshots for all our distributables (#181363)" This reverts commit 26b8c71730de6686fe4fe5de5f60ab5577b79902. --- config/node.options | 4 ---- .../os_packages/docker_generator/templates/base/Dockerfile | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config/node.options b/config/node.options index 2bc49f5db1f4ab..abcb40a5c19d4a 100644 --- a/config/node.options +++ b/config/node.options @@ -13,7 +13,3 @@ ## enable OpenSSL 3 legacy provider --openssl-legacy-provider - -# Enable capturing heap snapshots. See https://nodejs.org/api/cli.html#--heapsnapshot-signalsignal ---heapsnapshot-signal=SIGUSR2 ---diagnostic-dir=./data \ No newline at end of file diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile index 2284e504229a81..1869086b51ab78 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile +++ b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile @@ -157,6 +157,9 @@ COPY --chown=1000:0 config/serverless.yml /usr/share/kibana/config/serverless.ym COPY --chown=1000:0 config/serverless.es.yml /usr/share/kibana/config/serverless.es.yml COPY --chown=1000:0 config/serverless.oblt.yml /usr/share/kibana/config/serverless.oblt.yml COPY --chown=1000:0 config/serverless.security.yml /usr/share/kibana/config/serverless.security.yml +# Supportability enhancement: enable capturing heap snapshots. See https://nodejs.org/api/cli.html#--heapsnapshot-signalsignal +RUN /usr/bin/echo -e '\n--heapsnapshot-signal=SIGUSR2' >> config/node.options +RUN /usr/bin/echo '--diagnostic-dir=./data' >> config/node.options ENV PROFILER_SIGNAL=SIGUSR1 {{/serverless}} {{^opensslLegacyProvider}} From 0bce10993fdf2d75bb2b1819b807654d5d5be777 Mon Sep 17 00:00:00 2001 From: christineweng <18648970+christineweng@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:58:50 -0500 Subject: [PATCH 066/138] [Security Solution][Alert Details] Fix ancestry and same source insights (#181095) ## Summary Commit f9259faa3d3f2dfbf824fe934f2a32ae1999cbe9 address https://github.com/elastic/kibana/issues/180842 - Fields like `kibana.alert.ancestors.id` and `kibana.alert.rule.parameters.index` were previously used to fetch insights, but they are specific to alerts. To enable alerts by ancestry and alerts by same source events for non-alerts, this PR adds event id and default indices as fall back. ![image](https://github.com/elastic/kibana/assets/18648970/6691f841-5906-48f6-af80-e99c03def092) Commit bf6f8b73080987067ad126056042e145c3da372b address https://github.com/elastic/kibana/issues/181237 - There are checks to guard whether we can show as least 1 insight ([here](https://github.com/elastic/kibana/blob/main/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.tsx#L59)). However, the guard is only checking whether we have valid parameters, there is still a possibility that 0 alert is returned. In that case, the correlations is blank. The alert flyout avoids this scenario because there is always at least 1 alert by same source - As part of the fix above, alert by same source is now always enabled. This PR ensures we show same source insight even though no alert is found. Before ![image](https://github.com/elastic/kibana/assets/18648970/f053bf11-644f-4a5a-bfeb-5e0ce574b84c) After ![image](https://github.com/elastic/kibana/assets/18648970/a7b4d568-4c1e-413a-8dc4-ecfefd9a6a74) **How to test** - Generate some events and alerts - Go to Explore -> Host -> Events table - Expand a row details, scroll down to insights -> correlations, alerts by same source and alerts by ancestry insights should be present for applicable events (if you have endpoint security rule enabled, filter by `event.kind==alert`) - Note you need premium and above to see alerts by ancestry insight. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../components/correlations_details.test.tsx | 37 +++++-------------- .../left/components/correlations_details.tsx | 27 +++++--------- ...lated_alerts_by_same_source_event.test.tsx | 13 +++++-- .../related_alerts_by_same_source_event.tsx | 6 +-- .../components/correlations_overview.test.tsx | 35 +++++++----------- .../components/correlations_overview.tsx | 33 ++++++++--------- .../components/insights_section.test.tsx | 7 ++++ ...lated_alerts_by_same_source_event.test.tsx | 11 ++++-- .../related_alerts_by_same_source_event.tsx | 3 +- ...e_show_related_alerts_by_ancestry.test.tsx | 12 ------ .../use_show_related_alerts_by_ancestry.ts | 26 +------------ ...lated_alerts_by_same_source_event.test.tsx | 14 +++++-- ...how_related_alerts_by_same_source_event.ts | 15 +++++--- 13 files changed, 97 insertions(+), 142 deletions(-) diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.test.tsx index aad62c152773ab..a0a147d9754d5e 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.test.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.test.tsx @@ -27,6 +27,7 @@ import { useFetchRelatedAlertsByAncestry } from '../../shared/hooks/use_fetch_re import { useFetchRelatedAlertsBySameSourceEvent } from '../../shared/hooks/use_fetch_related_alerts_by_same_source_event'; import { useFetchRelatedCases } from '../../shared/hooks/use_fetch_related_cases'; import { mockContextValue } from '../mocks/mock_context'; +import { useTimelineDataFilters } from '../../../../timelines/containers/use_timeline_data_filters'; import { EXPANDABLE_PANEL_HEADER_TITLE_TEXT_TEST_ID } from '../../../shared/components/test_ids'; jest.mock('react-router-dom', () => { @@ -43,6 +44,11 @@ jest.mock('../../shared/hooks/use_fetch_related_alerts_by_ancestry'); jest.mock('../../shared/hooks/use_fetch_related_alerts_by_same_source_event'); jest.mock('../../shared/hooks/use_fetch_related_cases'); +jest.mock('../../../../timelines/containers/use_timeline_data_filters', () => ({ + useTimelineDataFilters: jest.fn(), +})); +const mockUseTimelineDataFilters = useTimelineDataFilters as jest.Mock; + const renderCorrelationDetails = () => { return render( @@ -62,12 +68,13 @@ const NO_DATA_MESSAGE = 'No correlations data available.'; describe('CorrelationsDetails', () => { beforeEach(() => { jest.clearAllMocks(); + mockUseTimelineDataFilters.mockReturnValue({ selectedPatterns: ['index'] }); }); it('renders all sections', () => { jest .mocked(useShowRelatedAlertsByAncestry) - .mockReturnValue({ show: true, documentId: 'event-id', indices: ['index1'] }); + .mockReturnValue({ show: true, documentId: 'event-id' }); jest .mocked(useShowRelatedAlertsBySameSourceEvent) .mockReturnValue({ show: true, originalEventId: 'originalEventId' }); @@ -115,7 +122,7 @@ describe('CorrelationsDetails', () => { it('should render no section and show error message if show values are false', () => { jest .mocked(useShowRelatedAlertsByAncestry) - .mockReturnValue({ show: false, documentId: 'event-id', indices: ['index1'] }); + .mockReturnValue({ show: false, documentId: 'event-id' }); jest .mocked(useShowRelatedAlertsBySameSourceEvent) .mockReturnValue({ show: false, originalEventId: 'originalEventId' }); @@ -142,30 +149,4 @@ describe('CorrelationsDetails', () => { ).not.toBeInTheDocument(); expect(getByText(NO_DATA_MESSAGE)).toBeInTheDocument(); }); - - it('should render no section if values are null', () => { - jest - .mocked(useShowRelatedAlertsByAncestry) - .mockReturnValue({ show: true, documentId: 'event-id' }); - jest.mocked(useShowRelatedAlertsBySameSourceEvent).mockReturnValue({ show: true }); - jest.mocked(useShowRelatedAlertsBySession).mockReturnValue({ show: true }); - jest.mocked(useShowRelatedCases).mockReturnValue(false); - jest.mocked(useShowSuppressedAlerts).mockReturnValue({ show: false, alertSuppressionCount: 0 }); - - const { queryByTestId } = renderCorrelationDetails(); - - expect( - queryByTestId(CORRELATIONS_DETAILS_BY_ANCESTRY_SECTION_TABLE_TEST_ID) - ).not.toBeInTheDocument(); - expect( - queryByTestId(CORRELATIONS_DETAILS_BY_SOURCE_SECTION_TABLE_TEST_ID) - ).not.toBeInTheDocument(); - expect( - queryByTestId(CORRELATIONS_DETAILS_BY_SESSION_SECTION_TABLE_TEST_ID) - ).not.toBeInTheDocument(); - expect(queryByTestId(CORRELATIONS_DETAILS_CASES_SECTION_TABLE_TEST_ID)).not.toBeInTheDocument(); - expect( - queryByTestId(CORRELATIONS_DETAILS_SUPPRESSED_ALERTS_TITLE_TEST_ID) - ).not.toBeInTheDocument(); - }); }); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.tsx index 226fbe4d7a4d6b..9c5a33a04a2434 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/correlations_details.tsx @@ -20,6 +20,8 @@ import { useShowRelatedAlertsBySameSourceEvent } from '../../shared/hooks/use_sh import { useShowRelatedAlertsBySession } from '../../shared/hooks/use_show_related_alerts_by_session'; import { RelatedAlertsByAncestry } from './related_alerts_by_ancestry'; import { SuppressedAlerts } from './suppressed_alerts'; +import { useTimelineDataFilters } from '../../../../timelines/containers/use_timeline_data_filters'; +import { isActiveTimeline } from '../../../../helpers'; export const CORRELATIONS_TAB_ID = 'correlations'; @@ -27,27 +29,18 @@ export const CORRELATIONS_TAB_ID = 'correlations'; * Correlations displayed in the document details expandable flyout left section under the Insights tab */ export const CorrelationsDetails: React.FC = () => { - const { - dataAsNestedObject, - dataFormattedForFieldBrowser, - eventId, - getFieldsData, - scopeId, - isPreview, - } = useLeftPanelContext(); + const { dataAsNestedObject, eventId, getFieldsData, scopeId, isPreview } = useLeftPanelContext(); + + const { selectedPatterns } = useTimelineDataFilters(isActiveTimeline(scopeId)); - const { - show: showAlertsByAncestry, - indices, - documentId, - } = useShowRelatedAlertsByAncestry({ + const { show: showAlertsByAncestry, documentId } = useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview, }); const { show: showSameSourceAlerts, originalEventId } = useShowRelatedAlertsBySameSourceEvent({ + eventId, getFieldsData, }); const { show: showAlertsBySession, entityId } = useShowRelatedAlertsBySession({ getFieldsData }); @@ -80,7 +73,7 @@ export const CorrelationsDetails: React.FC = () => { )} - {showSameSourceAlerts && originalEventId && ( + {showSameSourceAlerts && ( { )} - {showAlertsByAncestry && documentId && indices && ( + {showAlertsByAncestry && ( diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.test.tsx index 66902bd9bda34c..e8334613d1d242 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.test.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.test.tsx @@ -94,14 +94,21 @@ describe('', () => { expect(getByTestId(CORRELATIONS_DETAILS_BY_SOURCE_SECTION_TABLE_TEST_ID)).toBeInTheDocument(); }); - it('should render null if error', () => { + it('should render no data message if error', () => { (useFetchRelatedAlertsBySameSourceEvent as jest.Mock).mockReturnValue({ loading: false, error: true, + data: [], + dataCount: 0, + }); + (usePaginatedAlerts as jest.Mock).mockReturnValue({ + loading: false, + error: false, + data: [], }); - const { container } = renderRelatedAlertsBySameSourceEvent(); - expect(container).toBeEmptyDOMElement(); + const { getByText } = renderRelatedAlertsBySameSourceEvent(); + expect(getByText('No related source events.')).toBeInTheDocument(); }); it('should render no data message', () => { diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.tsx index e3bbf48c5fe151..42c9d910d93c68 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/left/components/related_alerts_by_same_source_event.tsx @@ -34,15 +34,11 @@ export const RelatedAlertsBySameSourceEvent: React.VFC { - const { loading, error, data, dataCount } = useFetchRelatedAlertsBySameSourceEvent({ + const { loading, data, dataCount } = useFetchRelatedAlertsBySameSourceEvent({ originalEventId, scopeId, }); - if (error) { - return null; - } - return ( ({ ExpandableFlyoutProvider: ({ children }: React.PropsWithChildren<{}>) => <>{children}, })); +jest.mock('../../../../timelines/containers/use_timeline_data_filters', () => ({ + useTimelineDataFilters: jest.fn(), +})); +const mockUseTimelineDataFilters = useTimelineDataFilters as jest.Mock; + +const originalEventId = 'originalEventId'; + describe('', () => { beforeAll(() => { jest.mocked(useExpandableFlyoutApi).mockReturnValue(flyoutContextValue); + mockUseTimelineDataFilters.mockReturnValue({ selectedPatterns: ['index'] }); }); it('should render wrapper component', () => { jest .mocked(useShowRelatedAlertsByAncestry) .mockReturnValue({ show: false, documentId: 'event-id' }); - jest.mocked(useShowRelatedAlertsBySameSourceEvent).mockReturnValue({ show: false }); + jest + .mocked(useShowRelatedAlertsBySameSourceEvent) + .mockReturnValue({ show: false, originalEventId }); jest.mocked(useShowRelatedAlertsBySession).mockReturnValue({ show: false }); jest.mocked(useShowRelatedCases).mockReturnValue(false); jest.mocked(useShowSuppressedAlerts).mockReturnValue({ show: false, alertSuppressionCount: 0 }); @@ -117,7 +128,7 @@ describe('', () => { it('should show component with all rows in expandable panel', () => { jest .mocked(useShowRelatedAlertsByAncestry) - .mockReturnValue({ show: true, documentId: 'event-id', indices: ['index1'] }); + .mockReturnValue({ show: true, documentId: 'event-id' }); jest .mocked(useShowRelatedAlertsBySameSourceEvent) .mockReturnValue({ show: true, originalEventId: 'originalEventId' }); @@ -160,7 +171,7 @@ describe('', () => { it('should hide rows and show error message if show values are false', () => { jest .mocked(useShowRelatedAlertsByAncestry) - .mockReturnValue({ show: false, documentId: 'event-id', indices: ['index1'] }); + .mockReturnValue({ show: false, documentId: 'event-id' }); jest .mocked(useShowRelatedAlertsBySameSourceEvent) .mockReturnValue({ show: false, originalEventId: 'originalEventId' }); @@ -179,24 +190,6 @@ describe('', () => { expect(getByText(NO_DATA_MESSAGE)).toBeInTheDocument(); }); - it('should hide rows if values are null', () => { - jest - .mocked(useShowRelatedAlertsByAncestry) - .mockReturnValue({ show: true, documentId: 'event-id' }); - jest.mocked(useShowRelatedAlertsBySameSourceEvent).mockReturnValue({ show: true }); - jest.mocked(useShowRelatedAlertsBySession).mockReturnValue({ show: true }); - jest.mocked(useShowRelatedCases).mockReturnValue(false); - jest.mocked(useShowSuppressedAlerts).mockReturnValue({ show: false, alertSuppressionCount: 0 }); - - const { queryByTestId, queryByText } = render(renderCorrelationsOverview(panelContextValue)); - expect(queryByTestId(RELATED_ALERTS_BY_ANCESTRY_TEST_ID)).not.toBeInTheDocument(); - expect(queryByTestId(RELATED_ALERTS_BY_SAME_SOURCE_EVENT_TEST_ID)).not.toBeInTheDocument(); - expect(queryByTestId(RELATED_ALERTS_BY_SESSION_TEST_ID)).not.toBeInTheDocument(); - expect(queryByTestId(RELATED_CASES_TEST_ID)).not.toBeInTheDocument(); - expect(queryByTestId(SUPPRESSED_ALERTS_TEST_ID)).not.toBeInTheDocument(); - expect(queryByText(NO_DATA_MESSAGE)).not.toBeInTheDocument(); - }); - it('should navigate to the left section Insights tab when clicking on button', () => { const { getByTestId } = render( diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/correlations_overview.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/correlations_overview.tsx index 6d91eabd003040..e272a058bf0da5 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/correlations_overview.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/correlations_overview.tsx @@ -24,6 +24,8 @@ import { CORRELATIONS_TEST_ID } from './test_ids'; import { useRightPanelContext } from '../context'; import { DocumentDetailsLeftPanelKey, LeftPanelInsightsTab } from '../../left'; import { CORRELATIONS_TAB_ID } from '../../left/components/correlations_details'; +import { useTimelineDataFilters } from '../../../../timelines/containers/use_timeline_data_filters'; +import { isActiveTimeline } from '../../../../helpers'; /** * Correlations section under Insights section, overview tab. @@ -31,17 +33,12 @@ import { CORRELATIONS_TAB_ID } from '../../left/components/correlations_details' * and the SummaryPanel component for data rendering. */ export const CorrelationsOverview: React.FC = () => { - const { - dataAsNestedObject, - dataFormattedForFieldBrowser, - eventId, - indexName, - getFieldsData, - scopeId, - isPreview, - } = useRightPanelContext(); + const { dataAsNestedObject, eventId, indexName, getFieldsData, scopeId, isPreview } = + useRightPanelContext(); const { openLeftPanel } = useExpandableFlyoutApi(); + const { selectedPatterns } = useTimelineDataFilters(isActiveTimeline(scopeId)); + const goToCorrelationsTab = useCallback(() => { openLeftPanel({ id: DocumentDetailsLeftPanelKey, @@ -57,18 +54,14 @@ export const CorrelationsOverview: React.FC = () => { }); }, [eventId, openLeftPanel, indexName, scopeId]); - const { - show: showAlertsByAncestry, - documentId, - indices, - } = useShowRelatedAlertsByAncestry({ + const { show: showAlertsByAncestry, documentId } = useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview, }); const { show: showSameSourceAlerts, originalEventId } = useShowRelatedAlertsBySameSourceEvent({ + eventId, getFieldsData, }); const { show: showAlertsBySession, entityId } = useShowRelatedAlertsBySession({ getFieldsData }); @@ -112,14 +105,18 @@ export const CorrelationsOverview: React.FC = () => { )} {showCases && } - {showSameSourceAlerts && originalEventId && ( + {showSameSourceAlerts && ( )} {showAlertsBySession && entityId && ( )} - {showAlertsByAncestry && documentId && indices && ( - + {showAlertsByAncestry && ( + )} ) : ( diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/insights_section.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/insights_section.test.tsx index 71897773d801e0..81f9c6d54457e0 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/insights_section.test.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/insights_section.test.tsx @@ -28,6 +28,7 @@ import { InsightsSection } from './insights_section'; import { useAlertPrevalence } from '../../../../common/containers/alerts/use_alert_prevalence'; import { useRiskScore } from '../../../../entity_analytics/api/hooks/use_risk_score'; import { useExpandSection } from '../hooks/use_expand_section'; +import { useTimelineDataFilters } from '../../../../timelines/containers/use_timeline_data_filters'; jest.mock('../../../../common/containers/alerts/use_alert_prevalence'); @@ -55,6 +56,11 @@ jest.mock('react-router-dom', () => { alertIds: [], }); +jest.mock('../../../../timelines/containers/use_timeline_data_filters', () => ({ + useTimelineDataFilters: jest.fn(), +})); +const mockUseTimelineDataFilters = useTimelineDataFilters as jest.Mock; + const from = '2022-04-05T12:00:00.000Z'; const to = '2022-04-08T12:00:00.;000Z'; const selectedPatterns = 'alerts'; @@ -101,6 +107,7 @@ const renderInsightsSection = (contextValue: RightPanelContext) => describe('', () => { beforeEach(() => { + mockUseTimelineDataFilters.mockReturnValue({ selectedPatterns: ['index'] }); mockUseUserDetails.mockReturnValue([false, { userDetails: null }]); mockUseRiskScore.mockReturnValue({ data: null, isAuthorized: false }); mockUseHostDetails.mockReturnValue([false, { hostDetails: null }]); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.test.tsx index 59dfb183a13380..d52d5473977893 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.test.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.test.tsx @@ -79,13 +79,18 @@ describe('', () => { expect(getByTestId(LOADING_TEST_ID)).toBeInTheDocument(); }); - it('should render null if error', () => { + it('should render 0 same source alert if error', () => { (useFetchRelatedAlertsBySameSourceEvent as jest.Mock).mockReturnValue({ loading: false, error: true, + dataCount: 0, }); - const { container } = renderRelatedAlertsBySameSourceEvent(); - expect(container).toBeEmptyDOMElement(); + const { getByTestId } = renderRelatedAlertsBySameSourceEvent(); + expect(getByTestId(ICON_TEST_ID)).toBeInTheDocument(); + const value = getByTestId(VALUE_TEST_ID); + expect(value).toBeInTheDocument(); + expect(value).toHaveTextContent('0 alerts related by source event'); + expect(getByTestId(VALUE_TEST_ID)).toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.tsx index e309a2e4bc93aa..0c1550dbb86920 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/related_alerts_by_same_source_event.tsx @@ -31,7 +31,7 @@ export const RelatedAlertsBySameSourceEvent: React.VFC { - const { loading, error, dataCount } = useFetchRelatedAlertsBySameSourceEvent({ + const { loading, dataCount } = useFetchRelatedAlertsBySameSourceEvent({ originalEventId, scopeId, }); @@ -46,7 +46,6 @@ export const RelatedAlertsBySameSourceEvent: React.VFC; const eventId = 'event-id'; const dataAsNestedObject = mockDataAsNestedObject; -const dataFormattedForFieldBrowser = mockDataFormattedForFieldBrowser; describe('useShowRelatedAlertsByAncestry', () => { let hookResult: RenderHookResult< @@ -53,7 +51,6 @@ describe('useShowRelatedAlertsByAncestry', () => { useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview: false, }) @@ -62,7 +59,6 @@ describe('useShowRelatedAlertsByAncestry', () => { expect(hookResult.result.current).toEqual({ show: false, documentId: 'event-id', - indices: ['rule-parameters-index'], }); }); @@ -74,7 +70,6 @@ describe('useShowRelatedAlertsByAncestry', () => { useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview: false, }) @@ -83,7 +78,6 @@ describe('useShowRelatedAlertsByAncestry', () => { expect(hookResult.result.current).toEqual({ show: false, documentId: 'event-id', - indices: ['rule-parameters-index'], }); }); @@ -95,7 +89,6 @@ describe('useShowRelatedAlertsByAncestry', () => { useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview: false, }) @@ -104,7 +97,6 @@ describe('useShowRelatedAlertsByAncestry', () => { expect(hookResult.result.current).toEqual({ show: false, documentId: 'event-id', - indices: ['rule-parameters-index'], }); }); @@ -117,7 +109,6 @@ describe('useShowRelatedAlertsByAncestry', () => { useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview: false, }) @@ -126,7 +117,6 @@ describe('useShowRelatedAlertsByAncestry', () => { expect(hookResult.result.current).toEqual({ show: true, documentId: 'event-id', - indices: ['rule-parameters-index'], }); }); @@ -139,7 +129,6 @@ describe('useShowRelatedAlertsByAncestry', () => { useShowRelatedAlertsByAncestry({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview: true, }) @@ -148,7 +137,6 @@ describe('useShowRelatedAlertsByAncestry', () => { expect(hookResult.result.current).toEqual({ show: true, documentId: 'ancestors-id', - indices: ['rule-parameters-index'], }); }); }); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_ancestry.ts b/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_ancestry.ts index 5a3f9c4f0b6578..b11485f67ae6eb 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_ancestry.ts +++ b/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_ancestry.ts @@ -6,14 +6,11 @@ */ import type { EcsSecurityExtension as Ecs } from '@kbn/securitysolution-ecs'; -import { useMemo } from 'react'; -import { find } from 'lodash/fp'; -import type { TimelineEventsDetailsItem } from '@kbn/timelines-plugin/common'; import type { GetFieldsData } from '../../../../common/hooks/use_get_fields_data'; import { useIsInvestigateInResolverActionEnabled } from '../../../../detections/components/alerts_table/timeline_actions/investigate_in_resolver'; import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; import { useLicense } from '../../../../common/hooks/use_license'; -import { ANCESTOR_ID, RULE_PARAMETERS_INDEX } from '../constants/field_names'; +import { ANCESTOR_ID } from '../constants/field_names'; import { getField } from '../utils'; export interface UseShowRelatedAlertsByAncestryParams { @@ -25,10 +22,6 @@ export interface UseShowRelatedAlertsByAncestryParams { * An object with top level fields from the ECS object */ dataAsNestedObject: Ecs; - /** - * An array of field objects with category and value - */ - dataFormattedForFieldBrowser: TimelineEventsDetailsItem[]; /** * Id of the event document */ @@ -44,10 +37,6 @@ export interface UseShowRelatedAlertsByAncestryResult { * Returns true if the user has at least platinum privilege, and if the document has ancestry data */ show: boolean; - /** - * Values of the kibana.alert.rule.parameters.index field - */ - indices?: string[]; /** * Value of the document id for fetching ancestry alerts */ @@ -60,7 +49,6 @@ export interface UseShowRelatedAlertsByAncestryResult { export const useShowRelatedAlertsByAncestry = ({ getFieldsData, dataAsNestedObject, - dataFormattedForFieldBrowser, eventId, isPreview, }: UseShowRelatedAlertsByAncestryParams): UseShowRelatedAlertsByAncestryResult => { @@ -71,24 +59,14 @@ export const useShowRelatedAlertsByAncestry = ({ const ancestorId = getField(getFieldsData(ANCESTOR_ID)) ?? ''; const documentId = isPreview ? ancestorId : eventId; - // can't use getFieldsData here as the kibana.alert.rule.parameters is different and can be nested - const originalDocumentIndex = useMemo( - () => find({ category: 'kibana', field: RULE_PARAMETERS_INDEX }, dataFormattedForFieldBrowser), - [dataFormattedForFieldBrowser] - ); const hasAtLeastPlatinum = useLicense().isPlatinumPlus(); const show = - isRelatedAlertsByProcessAncestryEnabled && - hasProcessEntityInfo && - originalDocumentIndex != null && - hasAtLeastPlatinum; + isRelatedAlertsByProcessAncestryEnabled && hasProcessEntityInfo && hasAtLeastPlatinum; return { show, documentId, - ...(originalDocumentIndex && - originalDocumentIndex.values && { indices: originalDocumentIndex.values }), }; }; diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.test.tsx index 01a014409264c9..dfbfeeccc655aa 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.test.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.test.tsx @@ -14,22 +14,28 @@ import type { } from './use_show_related_alerts_by_same_source_event'; import { useShowRelatedAlertsBySameSourceEvent } from './use_show_related_alerts_by_same_source_event'; +const eventId = 'eventId'; + describe('useShowRelatedAlertsBySameSourceEvent', () => { let hookResult: RenderHookResult< ShowRelatedAlertsBySameSourceEventParams, ShowRelatedAlertsBySameSourceEventResult >; - it('should return false if getFieldsData returns null', () => { + it('should return eventId if getFieldsData returns null', () => { const getFieldsData = () => null; - hookResult = renderHook(() => useShowRelatedAlertsBySameSourceEvent({ getFieldsData })); + hookResult = renderHook(() => + useShowRelatedAlertsBySameSourceEvent({ getFieldsData, eventId }) + ); - expect(hookResult.result.current).toEqual({ show: false }); + expect(hookResult.result.current).toEqual({ show: true, originalEventId: 'eventId' }); }); it('should return true if getFieldsData has the correct field', () => { const getFieldsData = () => 'original_event'; - hookResult = renderHook(() => useShowRelatedAlertsBySameSourceEvent({ getFieldsData })); + hookResult = renderHook(() => + useShowRelatedAlertsBySameSourceEvent({ getFieldsData, eventId }) + ); expect(hookResult.result.current).toEqual({ show: true, originalEventId: 'original_event' }); }); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.ts b/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.ts index 0d510400d5efea..2f76c74b329d1e 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.ts +++ b/x-pack/plugins/security_solution/public/flyout/document_details/shared/hooks/use_show_related_alerts_by_same_source_event.ts @@ -10,6 +10,10 @@ import { ANCESTOR_ID } from '../constants/field_names'; import { getField } from '../utils'; export interface ShowRelatedAlertsBySameSourceEventParams { + /** + * Id of the event document + */ + eventId: string; /** * Retrieves searchHit values for the provided field */ @@ -24,18 +28,19 @@ export interface ShowRelatedAlertsBySameSourceEventResult { /** * Value of the kibana.alert.original_event.id field */ - originalEventId?: string; + originalEventId: string; } /** - * Returns true if document has kibana.alert.original.event.id field with values + * Returns kibana.alert.ancestors.id field or default eventId */ export const useShowRelatedAlertsBySameSourceEvent = ({ + eventId, getFieldsData, }: ShowRelatedAlertsBySameSourceEventParams): ShowRelatedAlertsBySameSourceEventResult => { - const originalEventId = getField(getFieldsData(ANCESTOR_ID)); + const originalEventId = getField(getFieldsData(ANCESTOR_ID)) ?? eventId; return { - show: originalEventId != null, - ...(originalEventId && { originalEventId }), + show: true, + originalEventId, }; }; From 60ab2cfe956dc824b336f115a3a0fe91ee093a9d Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 24 Apr 2024 14:05:19 -0600 Subject: [PATCH 067/138] PresentationPanel props hidePanelChrome (#181473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Changes 1. adds `hidePanelChrome` parameter to `ReactEmbeddableRenderer`. When true, embeddable is rendered without `PresentationPanel` wrapper 2. Removes `embeddable-explorer` plugin 3. Moves Embeddable developer example into embeddable_examples plugin 4. Creates new examples that demonstrate how to use `ReactEmbeddableRenderer` Screenshot 2024-04-23 at 5 19 18 PM Follow-up work to narrow scope of this PR 1. add key concepts to embeddable overview 2. add "Register new embeddable type" tab that details how to create a new embeddable and shows how you can add embeddable examples to dashboard 3. group embeddable examples into a single item in "Add panel" menu - to show best practices of how to keep menu clean 4. remove class based example embeddables ### Test instructions 1. start kibana with `yarn start --run-examples` 5. Open kibana menu and click "Developer examples" 6. Click "Embeddables" card and run examples --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 - examples/embeddable_examples/kibana.jsonc | 3 +- .../embeddable_examples/public/app/app.tsx | 81 +++++++++++ .../public/app/overview.tsx} | 15 +- .../public/app/render_examples.tsx | 137 ++++++++++++++++++ .../public/app/setup_app.ts | 31 ++++ examples/embeddable_examples/public/plugin.ts | 39 ++--- .../search/search_embeddable_renderer.tsx | 53 +++++++ .../public/react_embeddables/search/types.ts | 3 + examples/embeddable_examples/tsconfig.json | 4 +- examples/embeddable_explorer/README.md | 10 -- examples/embeddable_explorer/kibana.jsonc | 20 --- examples/embeddable_explorer/public/app.tsx | 123 ---------------- .../public/embeddable_panel_example.tsx | 66 --------- .../public/embeddables.png | Bin 88399 -> 0 bytes .../public/hello_world_embeddable_example.tsx | 72 --------- .../public/list_container_example.tsx | 82 ----------- .../embeddable_explorer/public/plugin.tsx | 82 ----------- examples/embeddable_explorer/tsconfig.json | 25 ---- package.json | 1 - .../interfaces/fetch/fetch.ts | 22 +-- .../react_embeddable_renderer.tsx | 10 +- .../panel_component/presentation_panel.tsx | 48 ++++-- test/examples/config.js | 1 - .../embeddables/hello_world_embeddable.ts | 37 ----- test/examples/embeddables/index.ts | 29 ---- test/examples/embeddables/list_container.ts | 32 ---- tsconfig.base.json | 2 - yarn.lock | 4 - 29 files changed, 392 insertions(+), 641 deletions(-) create mode 100644 examples/embeddable_examples/public/app/app.tsx rename examples/{embeddable_explorer/public/index.ts => embeddable_examples/public/app/overview.tsx} (50%) create mode 100644 examples/embeddable_examples/public/app/render_examples.tsx create mode 100644 examples/embeddable_examples/public/app/setup_app.ts create mode 100644 examples/embeddable_examples/public/react_embeddables/search/search_embeddable_renderer.tsx delete mode 100644 examples/embeddable_explorer/README.md delete mode 100644 examples/embeddable_explorer/kibana.jsonc delete mode 100644 examples/embeddable_explorer/public/app.tsx delete mode 100644 examples/embeddable_explorer/public/embeddable_panel_example.tsx delete mode 100644 examples/embeddable_explorer/public/embeddables.png delete mode 100644 examples/embeddable_explorer/public/hello_world_embeddable_example.tsx delete mode 100644 examples/embeddable_explorer/public/list_container_example.tsx delete mode 100644 examples/embeddable_explorer/public/plugin.tsx delete mode 100644 examples/embeddable_explorer/tsconfig.json delete mode 100644 test/examples/embeddables/hello_world_embeddable.ts delete mode 100644 test/examples/embeddables/index.ts delete mode 100644 test/examples/embeddables/list_container.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0c57ef0d7dc9a0..0f584440699689 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -384,7 +384,6 @@ test/plugin_functional/plugins/elasticsearch_client_plugin @elastic/kibana-core x-pack/test/plugin_api_integration/plugins/elasticsearch_client @elastic/kibana-core x-pack/plugins/embeddable_enhanced @elastic/kibana-presentation examples/embeddable_examples @elastic/kibana-presentation -examples/embeddable_explorer @elastic/kibana-presentation src/plugins/embeddable @elastic/kibana-presentation x-pack/examples/embedded_lens_example @elastic/kibana-visualizations x-pack/plugins/encrypted_saved_objects @elastic/kibana-security diff --git a/examples/embeddable_examples/kibana.jsonc b/examples/embeddable_examples/kibana.jsonc index 0788268aedf3f5..08e4a02360b2c0 100644 --- a/examples/embeddable_examples/kibana.jsonc +++ b/examples/embeddable_examples/kibana.jsonc @@ -14,7 +14,8 @@ "dashboard", "data", "charts", - "fieldFormats" + "fieldFormats", + "developerExamples" ], "requiredBundles": ["presentationUtil"], "extraPublicDirs": ["public/hello_world"] diff --git a/examples/embeddable_examples/public/app/app.tsx b/examples/embeddable_examples/public/app/app.tsx new file mode 100644 index 00000000000000..72c5d4f11779a2 --- /dev/null +++ b/examples/embeddable_examples/public/app/app.tsx @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { useState } from 'react'; +import ReactDOM from 'react-dom'; + +import { AppMountParameters } from '@kbn/core-application-browser'; +import { + EuiPage, + EuiPageBody, + EuiPageHeader, + EuiPageSection, + EuiPageTemplate, + EuiSpacer, + EuiTab, + EuiTabs, +} from '@elastic/eui'; +import { Overview } from './overview'; +import { RenderExamples } from './render_examples'; + +const OVERVIEW_TAB_ID = 'overview'; +const RENDER_TAB_ID = 'render'; + +const App = () => { + const [selectedTabId, setSelectedTabId] = useState(OVERVIEW_TAB_ID); + + function onSelectedTabChanged(tabId: string) { + setSelectedTabId(tabId); + } + + function renderTabContent() { + if (selectedTabId === RENDER_TAB_ID) { + return ; + } + + return ; + } + + return ( + + + + + + + + + onSelectedTabChanged(OVERVIEW_TAB_ID)} + isSelected={OVERVIEW_TAB_ID === selectedTabId} + > + Embeddables overview + + onSelectedTabChanged(RENDER_TAB_ID)} + isSelected={RENDER_TAB_ID === selectedTabId} + > + Rendering embeddables in your application + + + + + + {renderTabContent()} + + + + + ); +}; + +export const renderApp = (element: AppMountParameters['element']) => { + ReactDOM.render(, element); + + return () => ReactDOM.unmountComponentAtNode(element); +}; diff --git a/examples/embeddable_explorer/public/index.ts b/examples/embeddable_examples/public/app/overview.tsx similarity index 50% rename from examples/embeddable_explorer/public/index.ts rename to examples/embeddable_examples/public/app/overview.tsx index 7620d1daf20f62..e074cd148f77e1 100644 --- a/examples/embeddable_explorer/public/index.ts +++ b/examples/embeddable_examples/public/app/overview.tsx @@ -6,6 +6,17 @@ * Side Public License, v 1. */ -import { EmbeddableExplorerPlugin } from './plugin'; +import React from 'react'; -export const plugin = () => new EmbeddableExplorerPlugin(); +import { EuiText } from '@elastic/eui'; + +export const Overview = () => { + return ( + +

+ Embeddables are React components that manage their own state, can be serialized and + deserialized, and return an API that can be used to interact with them imperatively. +

+ + ); +}; diff --git a/examples/embeddable_examples/public/app/render_examples.tsx b/examples/embeddable_examples/public/app/render_examples.tsx new file mode 100644 index 00000000000000..c3e9b0a79a55b6 --- /dev/null +++ b/examples/embeddable_examples/public/app/render_examples.tsx @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { useMemo, useState } from 'react'; + +import { ReactEmbeddableRenderer } from '@kbn/embeddable-plugin/public'; +import { + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiSuperDatePicker, + EuiSwitch, + EuiText, + OnTimeChangeProps, +} from '@elastic/eui'; +import { BehaviorSubject, Subject } from 'rxjs'; +import { TimeRange } from '@kbn/es-query'; +import { useBatchedOptionalPublishingSubjects } from '@kbn/presentation-publishing'; +import { SearchEmbeddableRenderer } from '../react_embeddables/search/search_embeddable_renderer'; +import { SEARCH_EMBEDDABLE_ID } from '../react_embeddables/search/constants'; +import type { Api, State } from '../react_embeddables/search/types'; + +export const RenderExamples = () => { + const initialState = useMemo(() => { + return { + rawState: { + timeRange: undefined, + }, + references: [], + }; + // only run onMount + }, []); + + const parentApi = useMemo(() => { + return { + reload$: new Subject(), + timeRange$: new BehaviorSubject({ + from: 'now-24h', + to: 'now', + }), + }; + // only run onMount + }, []); + + const [api, setApi] = useState(null); + const [hidePanelChrome, setHidePanelChrome] = useState(false); + const [dataLoading, timeRange] = useBatchedOptionalPublishingSubjects( + api?.dataLoading, + parentApi.timeRange$ + ); + + return ( +
+ { + parentApi.timeRange$.next({ + from: start, + to: end, + }); + }} + onRefresh={() => { + parentApi.reload$.next(); + }} + /> + + + + + + +

+ Use ReactEmbeddableRenderer to render embeddables. +

+ + + + {` + type={SEARCH_EMBEDDABLE_ID} + state={initialState} + parentApi={parentApi} + onApiAvailable={(newApi) => { + setApi(newApi); + }} + hidePanelChrome={hidePanelChrome} +/>`} + + + + + setHidePanelChrome(e.target.checked)} + /> + + + + + key={hidePanelChrome ? 'hideChrome' : 'showChrome'} + type={SEARCH_EMBEDDABLE_ID} + state={initialState} + parentApi={parentApi} + onApiAvailable={(newApi) => { + setApi(newApi); + }} + hidePanelChrome={hidePanelChrome} + /> + + + + +

To avoid leaking embeddable details, wrap ReactEmbeddableRenderer in a component.

+ + + + {``} + + + + + + + +
+ ); +}; diff --git a/examples/embeddable_examples/public/app/setup_app.ts b/examples/embeddable_examples/public/app/setup_app.ts new file mode 100644 index 00000000000000..c489b603c877b7 --- /dev/null +++ b/examples/embeddable_examples/public/app/setup_app.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { AppMountParameters, CoreSetup } from '@kbn/core/public'; +import { DeveloperExamplesSetup } from '@kbn/developer-examples-plugin/public'; +import type { StartDeps } from '../plugin'; + +const APP_ID = 'embeddablesApp'; +const title = 'Embeddables'; + +export function setupApp(core: CoreSetup, developerExamples: DeveloperExamplesSetup) { + core.application.register({ + id: APP_ID, + title, + visibleIn: [], + async mount(params: AppMountParameters) { + const { renderApp } = await import('./app'); + return renderApp(params.element); + }, + }); + developerExamples.register({ + appId: APP_ID, + title, + description: `Learn how to create new embeddable types and use embeddables in your application.`, + }); +} diff --git a/examples/embeddable_examples/public/plugin.ts b/examples/embeddable_examples/public/plugin.ts index e3916b8b409cb9..85faad072920a6 100644 --- a/examples/embeddable_examples/public/plugin.ts +++ b/examples/embeddable_examples/public/plugin.ts @@ -17,6 +17,7 @@ import { UiActionsStart } from '@kbn/ui-actions-plugin/public'; import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { ChartsPluginStart } from '@kbn/charts-plugin/public'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; +import { DeveloperExamplesSetup } from '@kbn/developer-examples-plugin/public'; import { HelloWorldEmbeddableFactory, HELLO_WORLD_EMBEDDABLE, @@ -45,13 +46,15 @@ import { registerAddSearchPanelAction } from './react_embeddables/search/registe import { EUI_MARKDOWN_ID } from './react_embeddables/eui_markdown/constants'; import { FIELD_LIST_ID } from './react_embeddables/field_list/constants'; import { SEARCH_EMBEDDABLE_ID } from './react_embeddables/search/constants'; +import { setupApp } from './app/setup_app'; -export interface EmbeddableExamplesSetupDependencies { +export interface SetupDeps { + developerExamples: DeveloperExamplesSetup; embeddable: EmbeddableSetup; uiActions: UiActionsStart; } -export interface EmbeddableExamplesStartDependencies { +export interface StartDeps { dataViews: DataViewsPublicPluginStart; embeddable: EmbeddableStart; uiActions: UiActionsStart; @@ -67,40 +70,31 @@ interface ExampleEmbeddableFactories { getFilterDebuggerEmbeddableFactory: () => FilterDebuggerEmbeddableFactory; } -export interface EmbeddableExamplesStart { +export interface StartApi { createSampleData: () => Promise; factories: ExampleEmbeddableFactories; } -export class EmbeddableExamplesPlugin - implements - Plugin< - void, - EmbeddableExamplesStart, - EmbeddableExamplesSetupDependencies, - EmbeddableExamplesStartDependencies - > -{ +export class EmbeddableExamplesPlugin implements Plugin { private exampleEmbeddableFactories: Partial = {}; - public setup( - core: CoreSetup, - deps: EmbeddableExamplesSetupDependencies - ) { + public setup(core: CoreSetup, { embeddable, developerExamples }: SetupDeps) { + setupApp(core, developerExamples); + this.exampleEmbeddableFactories.getHelloWorldEmbeddableFactory = - deps.embeddable.registerEmbeddableFactory( + embeddable.registerEmbeddableFactory( HELLO_WORLD_EMBEDDABLE, new HelloWorldEmbeddableFactoryDefinition() ); this.exampleEmbeddableFactories.getMigrationsEmbeddableFactory = - deps.embeddable.registerEmbeddableFactory( + embeddable.registerEmbeddableFactory( SIMPLE_EMBEDDABLE, new SimpleEmbeddableFactoryDefinition() ); this.exampleEmbeddableFactories.getListContainerEmbeddableFactory = - deps.embeddable.registerEmbeddableFactory( + embeddable.registerEmbeddableFactory( LIST_CONTAINER, new ListContainerFactoryDefinition(async () => ({ embeddableServices: (await core.getStartServices())[1].embeddable, @@ -108,16 +102,13 @@ export class EmbeddableExamplesPlugin ); this.exampleEmbeddableFactories.getFilterDebuggerEmbeddableFactory = - deps.embeddable.registerEmbeddableFactory( + embeddable.registerEmbeddableFactory( FILTER_DEBUGGER_EMBEDDABLE, new FilterDebuggerEmbeddableFactoryDefinition() ); } - public start( - core: CoreStart, - deps: EmbeddableExamplesStartDependencies - ): EmbeddableExamplesStart { + public start(core: CoreStart, deps: StartDeps): StartApi { registerCreateFieldListAction(deps.uiActions); registerReactEmbeddableFactory(FIELD_LIST_ID, async () => { const { getFieldListFactory } = await import( diff --git a/examples/embeddable_examples/public/react_embeddables/search/search_embeddable_renderer.tsx b/examples/embeddable_examples/public/react_embeddables/search/search_embeddable_renderer.tsx new file mode 100644 index 00000000000000..0fa6e785b72c17 --- /dev/null +++ b/examples/embeddable_examples/public/react_embeddables/search/search_embeddable_renderer.tsx @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { useEffect, useMemo } from 'react'; +import { BehaviorSubject } from 'rxjs'; +import { TimeRange } from '@kbn/es-query'; +import { ReactEmbeddableRenderer } from '@kbn/embeddable-plugin/public'; +import type { Api, State } from './types'; +import { SEARCH_EMBEDDABLE_ID } from './constants'; + +interface Props { + timeRange?: TimeRange; +} + +export function SearchEmbeddableRenderer(props: Props) { + const initialState = useMemo(() => { + return { + rawState: { + timeRange: undefined, + }, + references: [], + }; + // only run onMount + }, []); + + const parentApi = useMemo(() => { + return { + timeRange$: new BehaviorSubject(props.timeRange), + }; + // only run onMount + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + parentApi.timeRange$.next(props.timeRange); + }, [props.timeRange, parentApi.timeRange$]); + + return ( +
+ + type={SEARCH_EMBEDDABLE_ID} + state={initialState} + parentApi={parentApi} + hidePanelChrome={true} + /> +
+ ); +} diff --git a/examples/embeddable_examples/public/react_embeddables/search/types.ts b/examples/embeddable_examples/public/react_embeddables/search/types.ts index 5ba940b2d40b0f..dffe119eee7a0a 100644 --- a/examples/embeddable_examples/public/react_embeddables/search/types.ts +++ b/examples/embeddable_examples/public/react_embeddables/search/types.ts @@ -19,6 +19,9 @@ import { } from '@kbn/presentation-publishing'; export interface State { + /* + * Time range only applied to this embeddable, overrides parentApi.timeRange$ + */ timeRange: TimeRange | undefined; } diff --git a/examples/embeddable_examples/tsconfig.json b/examples/embeddable_examples/tsconfig.json index 02250052627204..1165c05b189adb 100644 --- a/examples/embeddable_examples/tsconfig.json +++ b/examples/embeddable_examples/tsconfig.json @@ -29,6 +29,8 @@ "@kbn/core-lifecycle-browser", "@kbn/presentation-util-plugin", "@kbn/unified-field-list", - "@kbn/presentation-containers" + "@kbn/presentation-containers", + "@kbn/core-application-browser", + "@kbn/developer-examples-plugin" ] } diff --git a/examples/embeddable_explorer/README.md b/examples/embeddable_explorer/README.md deleted file mode 100644 index 0425790f07487c..00000000000000 --- a/examples/embeddable_explorer/README.md +++ /dev/null @@ -1,10 +0,0 @@ -## Embeddable explorer - -This example app shows how to: - - Create a basic "hello world" embeddable - - Create embeddables that accept inputs and use an EmbeddableRenderer - - Nest embeddables inside a container - - Dynamically add children to embeddable containers - - Work with the EmbeddablePanel component - -To run this example, use the command `yarn start --run-examples`. diff --git a/examples/embeddable_explorer/kibana.jsonc b/examples/embeddable_explorer/kibana.jsonc deleted file mode 100644 index 3279a7ef92f517..00000000000000 --- a/examples/embeddable_explorer/kibana.jsonc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "plugin", - "id": "@kbn/embeddable-explorer-plugin", - "owner": "@elastic/kibana-presentation", - "description": "Example app that relies on registered functionality in the embeddable_examples plugin", - "plugin": { - "id": "embeddableExplorer", - "server": false, - "browser": true, - "requiredPlugins": [ - "uiActions", - "inspector", - "embeddable", - "embeddableExamples", - "developerExamples", - "dashboard", - "kibanaReact" - ] - } -} diff --git a/examples/embeddable_explorer/public/app.tsx b/examples/embeddable_explorer/public/app.tsx deleted file mode 100644 index 1d1938c8dbebd5..00000000000000 --- a/examples/embeddable_explorer/public/app.tsx +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React from 'react'; -import ReactDOM from 'react-dom'; -import { BrowserRouter as Router, withRouter, RouteComponentProps } from 'react-router-dom'; -import { Route } from '@kbn/shared-ux-router'; -import { EuiPageTemplate, EuiSideNav } from '@elastic/eui'; -import { EmbeddableStart } from '@kbn/embeddable-plugin/public'; -import { UiActionsStart } from '@kbn/ui-actions-plugin/public'; -import { Start as InspectorStartContract } from '@kbn/inspector-plugin/public'; -import { AppMountParameters, CoreStart, IUiSettingsClient, OverlayStart } from '@kbn/core/public'; -import { EmbeddableExamplesStart } from '@kbn/embeddable-examples-plugin/public/plugin'; -import { HelloWorldEmbeddableExample } from './hello_world_embeddable_example'; -import { ListContainerExample } from './list_container_example'; -import { EmbeddablePanelExample } from './embeddable_panel_example'; - -interface PageDef { - title: string; - id: string; - component: React.ReactNode; -} - -type NavProps = RouteComponentProps & { - navigateToApp: CoreStart['application']['navigateToApp']; - pages: PageDef[]; -}; - -const Nav = withRouter(({ history, navigateToApp, pages }: NavProps) => { - const navItems = pages.map((page) => ({ - id: page.id, - name: page.title, - onClick: () => history.push(`/${page.id}`), - 'data-test-subj': page.id, - })); - - return ( - - ); -}); - -interface Props { - basename: string; - navigateToApp: CoreStart['application']['navigateToApp']; - embeddableApi: EmbeddableStart; - uiActionsApi: UiActionsStart; - overlays: OverlayStart; - notifications: CoreStart['notifications']; - inspector: InspectorStartContract; - uiSettingsClient: IUiSettingsClient; - embeddableExamples: EmbeddableExamplesStart; -} - -const EmbeddableExplorerApp = ({ - basename, - navigateToApp, - embeddableApi, - embeddableExamples, -}: Props) => { - const pages: PageDef[] = [ - { - title: 'Render embeddable', - id: 'helloWorldEmbeddableSection', - component: ( - - ), - }, - { - title: 'Groups of embeddables', - id: 'listContainerSection', - component: ( - - ), - }, - { - title: 'Context menu', - id: 'embeddablePanelExample', - component: ( - - ), - }, - ]; - - const routes = pages.map((page, i) => ( - page.component} /> - )); - - return ( - - - -
); diff --git a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx b/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx index c6297f6f56e2b8..a5647cb1c49f11 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx +++ b/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx @@ -6,14 +6,14 @@ * Side Public License, v 1. */ -import React, { FunctionComponent, useEffect, useState } from 'react'; +import React, { FunctionComponent, useCallback, useEffect, useRef, useState } from 'react'; import { CodeEditor } from '@kbn/code-editor'; import { css } from '@emotion/react'; import { VectorTile } from '@mapbox/vector-tile'; import Protobuf from 'pbf'; import { i18n } from '@kbn/i18n'; import { EuiScreenReaderOnly } from '@elastic/eui'; -import { CONSOLE_OUTPUT_THEME_ID, CONSOLE_OUTPUT_LANG_ID } from '@kbn/monaco'; +import { CONSOLE_OUTPUT_THEME_ID, CONSOLE_OUTPUT_LANG_ID, monaco } from '@kbn/monaco'; import { useEditorReadContext, useRequestReadContext } from '../../../contexts'; import { convertMapboxVectorTileToJson } from '../legacy/console_editor/mapbox_vector_tile'; import { @@ -22,6 +22,7 @@ import { safeExpandLiteralStrings, languageForContentType, } from '../utilities'; +import { useResizeCheckerUtils } from './use_resize_checker_utils'; export const MonacoEditorOutput: FunctionComponent = () => { const { settings: readOnlySettings } = useEditorReadContext(); @@ -30,6 +31,19 @@ export const MonacoEditorOutput: FunctionComponent = () => { } = useRequestReadContext(); const [value, setValue] = useState(''); const [mode, setMode] = useState('text'); + const divRef = useRef(null); + const { setupResizeChecker, destroyResizeChecker } = useResizeCheckerUtils(); + + const editorDidMountCallback = useCallback( + (editor: monaco.editor.IStandaloneCodeEditor) => { + setupResizeChecker(divRef.current!, editor); + }, + [setupResizeChecker] + ); + + const editorWillUnmountCallback = useCallback(() => { + destroyResizeChecker(); + }, [destroyResizeChecker]); useEffect(() => { if (data) { @@ -69,6 +83,7 @@ export const MonacoEditorOutput: FunctionComponent = () => { css={css` width: 100%; `} + ref={divRef} >
diff --git a/src/plugins/console/public/application/containers/editor/monaco/use_resize_checker_utils.ts b/src/plugins/console/public/application/containers/editor/monaco/use_resize_checker_utils.ts new file mode 100644 index 00000000000000..985fe47601fe48 --- /dev/null +++ b/src/plugins/console/public/application/containers/editor/monaco/use_resize_checker_utils.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { useRef } from 'react'; +import { ResizeChecker } from '@kbn/kibana-utils-plugin/public'; +import { monaco } from '@kbn/monaco'; + +/** + * Hook that returns functions for setting up and destroying a {@link ResizeChecker} + * for a Monaco editor. + */ +export const useResizeCheckerUtils = () => { + const resizeChecker = useRef(null); + + const setupResizeChecker = ( + divElement: HTMLDivElement, + editor: monaco.editor.IStandaloneCodeEditor + ) => { + if (resizeChecker.current) { + resizeChecker.current.destroy(); + } + resizeChecker.current = new ResizeChecker(divElement); + resizeChecker.current.on('resize', () => { + editor.layout(); + }); + }; + + const destroyResizeChecker = () => { + resizeChecker.current!.destroy(); + }; + + return { setupResizeChecker, destroyResizeChecker }; +}; From ec5b6fd2140cd3f082398a43f96a5dd1c2108307 Mon Sep 17 00:00:00 2001 From: Elena Stoeva <59341489+ElenaStoeva@users.noreply.github.com> Date: Thu, 25 Apr 2024 13:41:22 +0100 Subject: [PATCH 098/138] [Code Editor] Move theme to kbn-monaco package (#181417) ## Summary This PR moves the Monaco themes used in the Code Editor component to the `kbn-monaco` package to prevent the circular dependency between the `kbn-monaco` and `code_editor` packages that occurs in https://github.com/elastic/kibana/pull/180321 (where the code editor theme is reused from inside the `kbn-monaco` package). --------- Co-authored-by: Stratoula Kalafateli --- packages/kbn-monaco/index.ts | 7 ++ .../kbn-monaco/src/code_editor/constants.ts | 12 ++ packages/kbn-monaco/src/code_editor/index.ts | 21 ++++ packages/kbn-monaco/src/code_editor/theme.ts | 111 ++++++++++++++++++ packages/kbn-monaco/src/register_globals.ts | 14 +++ .../src/text_based_languages_editor.test.tsx | 17 +++ .../__snapshots__/code_editor.test.tsx.snap | 4 +- .../code_editor/impl/code_editor.tsx | 43 +++---- .../code_editor/impl/editor.styles.ts | 108 ----------------- .../template_clone.test.tsx | 16 +++ 10 files changed, 217 insertions(+), 136 deletions(-) create mode 100644 packages/kbn-monaco/src/code_editor/constants.ts create mode 100644 packages/kbn-monaco/src/code_editor/index.ts create mode 100644 packages/kbn-monaco/src/code_editor/theme.ts diff --git a/packages/kbn-monaco/index.ts b/packages/kbn-monaco/index.ts index 3ba4a5afb3aeac..fad82d4d4a1430 100644 --- a/packages/kbn-monaco/index.ts +++ b/packages/kbn-monaco/index.ts @@ -42,3 +42,10 @@ export { } from './src/console'; export type { ParsedRequest } from './src/console'; + +export { + CODE_EDITOR_LIGHT_THEME_ID, + CODE_EDITOR_DARK_THEME_ID, + CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID, + CODE_EDITOR_DARK_THEME_TRANSPARENT_ID, +} from './src/code_editor'; diff --git a/packages/kbn-monaco/src/code_editor/constants.ts b/packages/kbn-monaco/src/code_editor/constants.ts new file mode 100644 index 00000000000000..78d4becb5c62d3 --- /dev/null +++ b/packages/kbn-monaco/src/code_editor/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const CODE_EDITOR_LIGHT_THEME_ID = 'codeEditorLightTheme'; +export const CODE_EDITOR_DARK_THEME_ID = 'codeEditorDarkTheme'; +export const CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID = 'codeEditorLightTransparentTheme'; +export const CODE_EDITOR_DARK_THEME_TRANSPARENT_ID = 'codeEditorDarkTransparentTheme'; diff --git a/packages/kbn-monaco/src/code_editor/index.ts b/packages/kbn-monaco/src/code_editor/index.ts new file mode 100644 index 00000000000000..6fa363f0699be8 --- /dev/null +++ b/packages/kbn-monaco/src/code_editor/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { + CODE_EDITOR_LIGHT_THEME_ID, + CODE_EDITOR_DARK_THEME_ID, + CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID, + CODE_EDITOR_DARK_THEME_TRANSPARENT_ID, +} from './constants'; + +export { + buildLightTheme, + buildDarkTheme, + buildLightTransparentTheme, + buildDarkTransparentTheme, +} from './theme'; diff --git a/packages/kbn-monaco/src/code_editor/theme.ts b/packages/kbn-monaco/src/code_editor/theme.ts new file mode 100644 index 00000000000000..fce111bfdec930 --- /dev/null +++ b/packages/kbn-monaco/src/code_editor/theme.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { euiDarkVars as darkTheme, euiLightVars as lightTheme } from '@kbn/ui-theme'; +import { monaco } from '../..'; + +export function createTheme( + euiTheme: typeof darkTheme | typeof lightTheme, + selectionBackgroundColor: string, + backgroundColor?: string +): monaco.editor.IStandaloneThemeData { + return { + base: 'vs', + inherit: true, + rules: [ + { + token: '', + foreground: euiTheme.euiColorDarkestShade, + background: euiTheme.euiFormBackgroundColor, + }, + { token: 'invalid', foreground: euiTheme.euiColorAccentText }, + { token: 'emphasis', fontStyle: 'italic' }, + { token: 'strong', fontStyle: 'bold' }, + + { token: 'variable', foreground: euiTheme.euiColorPrimaryText }, + { token: 'variable.predefined', foreground: euiTheme.euiColorSuccessText }, + { token: 'constant', foreground: euiTheme.euiColorAccentText }, + { token: 'comment', foreground: euiTheme.euiTextSubduedColor }, + { token: 'number', foreground: euiTheme.euiColorAccentText }, + { token: 'number.hex', foreground: euiTheme.euiColorAccentText }, + { token: 'regexp', foreground: euiTheme.euiColorDangerText }, + { token: 'annotation', foreground: euiTheme.euiTextSubduedColor }, + { token: 'type', foreground: euiTheme.euiColorSuccessText }, + + { token: 'delimiter', foreground: euiTheme.euiTextSubduedColor }, + { token: 'delimiter.html', foreground: euiTheme.euiColorDarkShade }, + { token: 'delimiter.xml', foreground: euiTheme.euiColorPrimaryText }, + + { token: 'tag', foreground: euiTheme.euiColorDangerText }, + { token: 'tag.id.jade', foreground: euiTheme.euiColorPrimaryText }, + { token: 'tag.class.jade', foreground: euiTheme.euiColorPrimaryText }, + { token: 'meta.scss', foreground: euiTheme.euiColorAccentText }, + { token: 'metatag', foreground: euiTheme.euiColorSuccessText }, + { token: 'metatag.content.html', foreground: euiTheme.euiColorDangerText }, + { token: 'metatag.html', foreground: euiTheme.euiTextSubduedColor }, + { token: 'metatag.xml', foreground: euiTheme.euiTextSubduedColor }, + { token: 'metatag.php', fontStyle: 'bold' }, + + { token: 'key', foreground: euiTheme.euiColorWarningText }, + { token: 'string.key.json', foreground: euiTheme.euiColorDangerText }, + { token: 'string.value.json', foreground: euiTheme.euiColorPrimaryText }, + + { token: 'attribute.name', foreground: euiTheme.euiColorDangerText }, + { token: 'attribute.name.css', foreground: euiTheme.euiColorSuccessText }, + { token: 'attribute.value', foreground: euiTheme.euiColorPrimaryText }, + { token: 'attribute.value.number', foreground: euiTheme.euiColorWarningText }, + { token: 'attribute.value.unit', foreground: euiTheme.euiColorWarningText }, + { token: 'attribute.value.html', foreground: euiTheme.euiColorPrimaryText }, + { token: 'attribute.value.xml', foreground: euiTheme.euiColorPrimaryText }, + + { token: 'string', foreground: euiTheme.euiColorDangerText }, + { token: 'string.html', foreground: euiTheme.euiColorPrimaryText }, + { token: 'string.sql', foreground: euiTheme.euiColorDangerText }, + { token: 'string.yaml', foreground: euiTheme.euiColorPrimaryText }, + + { token: 'keyword', foreground: euiTheme.euiColorPrimaryText }, + { token: 'keyword.json', foreground: euiTheme.euiColorPrimaryText }, + { token: 'keyword.flow', foreground: euiTheme.euiColorWarningText }, + { token: 'keyword.flow.scss', foreground: euiTheme.euiColorPrimaryText }, + // Monaco editor supports strikethrough font style only starting from 0.32.0. + { token: 'keyword.deprecated', foreground: euiTheme.euiColorAccentText }, + + { token: 'operator.scss', foreground: euiTheme.euiColorDarkShade }, + { token: 'operator.sql', foreground: euiTheme.euiTextSubduedColor }, + { token: 'operator.swift', foreground: euiTheme.euiTextSubduedColor }, + { token: 'predefined.sql', foreground: euiTheme.euiTextSubduedColor }, + + { token: 'text', foreground: euiTheme.euiTitleColor }, + { token: 'label', foreground: euiTheme.euiColorVis9 }, + ], + colors: { + 'editor.foreground': euiTheme.euiColorDarkestShade, + 'editor.background': backgroundColor ?? euiTheme.euiFormBackgroundColor, + 'editorLineNumber.foreground': euiTheme.euiColorDarkShade, + 'editorLineNumber.activeForeground': euiTheme.euiColorDarkShade, + 'editorIndentGuide.background1': euiTheme.euiColorLightShade, + 'editor.selectionBackground': selectionBackgroundColor, + 'editorWidget.border': euiTheme.euiColorLightShade, + 'editorWidget.background': euiTheme.euiColorLightestShade, + 'editorCursor.foreground': euiTheme.euiColorDarkestShade, + 'editorSuggestWidget.selectedForeground': euiTheme.euiColorDarkestShade, + 'editorSuggestWidget.focusHighlightForeground': euiTheme.euiColorPrimary, + 'editorSuggestWidget.selectedBackground': euiTheme.euiColorLightShade, + 'list.hoverBackground': euiTheme.euiColorLightShade, + 'list.highlightForeground': euiTheme.euiColorPrimary, + 'editor.lineHighlightBorder': euiTheme.euiColorLightestShade, + 'editorHoverWidget.foreground': euiTheme.euiColorDarkestShade, + 'editorHoverWidget.background': euiTheme.euiFormBackgroundColor, + }, + }; +} + +export const buildDarkTheme = () => createTheme(darkTheme, '#343551'); +export const buildLightTheme = () => createTheme(lightTheme, '#E3E4ED'); +export const buildDarkTransparentTheme = () => createTheme(darkTheme, '#343551', '#00000000'); +export const buildLightTransparentTheme = () => createTheme(lightTheme, '#E3E4ED', '#00000000'); diff --git a/packages/kbn-monaco/src/register_globals.ts b/packages/kbn-monaco/src/register_globals.ts index 82c057b43ae424..7e12ee24b386a5 100644 --- a/packages/kbn-monaco/src/register_globals.ts +++ b/packages/kbn-monaco/src/register_globals.ts @@ -21,6 +21,16 @@ import { buildConsoleTheme, buildConsoleOutputTheme, } from './console'; +import { + CODE_EDITOR_LIGHT_THEME_ID, + CODE_EDITOR_DARK_THEME_ID, + CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID, + CODE_EDITOR_DARK_THEME_TRANSPARENT_ID, + buildLightTheme, + buildDarkTheme, + buildLightTransparentTheme, + buildDarkTransparentTheme, +} from './code_editor'; export const DEFAULT_WORKER_ID = 'default'; const langSpecificWorkerIds = [ @@ -49,6 +59,10 @@ registerLanguage(ConsoleOutputLang); registerTheme(ESQL_THEME_ID, buildESQlTheme()); registerTheme(CONSOLE_THEME_ID, buildConsoleTheme()); registerTheme(CONSOLE_OUTPUT_THEME_ID, buildConsoleOutputTheme()); +registerTheme(CODE_EDITOR_LIGHT_THEME_ID, buildLightTheme()); +registerTheme(CODE_EDITOR_DARK_THEME_ID, buildDarkTheme()); +registerTheme(CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID, buildLightTransparentTheme()); +registerTheme(CODE_EDITOR_DARK_THEME_TRANSPARENT_ID, buildDarkTransparentTheme()); const monacoBundleDir = (window as any).__kbnPublicPath__?.['kbn-monaco']; diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.test.tsx b/packages/kbn-text-based-editor/src/text_based_languages_editor.test.tsx index bbcb29e253981b..6aae293e9ae9bf 100644 --- a/packages/kbn-text-based-editor/src/text_based_languages_editor.test.tsx +++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.test.tsx @@ -59,6 +59,23 @@ describe('TextBasedLanguagesEditor', () => { ); } let props: TextBasedLanguagesEditorProps; + beforeAll(() => { + // Mocking matchMedia to resolve TypeError: window.matchMedia is not a function + // For more info, see https://jestjs.io/docs/manual-mocks#mocking-methods-which-are-not-implemented-in-jsdom + Object.defineProperty(window, 'matchMedia', { + writable: true, + value: jest.fn().mockImplementation((query) => ({ + matches: false, + media: query, + onchange: null, + addListener: jest.fn(), // deprecated + removeListener: jest.fn(), // deprecated + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + dispatchEvent: jest.fn(), + })), + }); + }); beforeEach(() => { props = { query: { esql: 'from test' }, diff --git a/packages/shared-ux/code_editor/impl/__snapshots__/code_editor.test.tsx.snap b/packages/shared-ux/code_editor/impl/__snapshots__/code_editor.test.tsx.snap index 9383b88dd43724..df22190992fb35 100644 --- a/packages/shared-ux/code_editor/impl/__snapshots__/code_editor.test.tsx.snap +++ b/packages/shared-ux/code_editor/impl/__snapshots__/code_editor.test.tsx.snap @@ -327,7 +327,7 @@ exports[` is rendered 1`] = ` "wrappingIndent": "indent", } } - theme="euiColors" + theme="codeEditorLightTheme" value=" [Sun Mar 7 20:54:27 2004] [notice] [client xx.xx.xx.xx] This is a notice! [Sun Mar 7 20:58:27 2004] [info] [client xx.xx.xx.xx] (104)Connection reset by peer: client stopped connection before send body completed @@ -365,7 +365,7 @@ exports[` is rendered 1`] = ` "wrappingIndent": "indent", } } - theme="euiColors" + theme="codeEditorLightTheme" value=" [Sun Mar 7 20:54:27 2004] [notice] [client xx.xx.xx.xx] This is a notice! [Sun Mar 7 20:58:27 2004] [info] [client xx.xx.xx.xx] (104)Connection reset by peer: client stopped connection before send body completed diff --git a/packages/shared-ux/code_editor/impl/code_editor.tsx b/packages/shared-ux/code_editor/impl/code_editor.tsx index 890e794afc7a47..2e8d2c2732ff73 100644 --- a/packages/shared-ux/code_editor/impl/code_editor.tsx +++ b/packages/shared-ux/code_editor/impl/code_editor.tsx @@ -23,7 +23,13 @@ import { EuiFlexItem, useEuiTheme, } from '@elastic/eui'; -import { monaco } from '@kbn/monaco'; +import { + monaco, + CODE_EDITOR_LIGHT_THEME_ID, + CODE_EDITOR_DARK_THEME_ID, + CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID, + CODE_EDITOR_DARK_THEME_TRANSPARENT_ID, +} from '@kbn/monaco'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { css } from '@emotion/react'; @@ -31,13 +37,7 @@ import './register_languages'; import { remeasureFonts } from './remeasure_fonts'; import { PlaceholderWidget } from './placeholder_widget'; -import { - styles, - DARK_THEME, - LIGHT_THEME, - DARK_THEME_TRANSPARENT, - LIGHT_THEME_TRANSPARENT, -} from './editor.styles'; +import { styles } from './editor.styles'; export interface CodeEditorProps { /** Width of editor. Defaults to 100%. */ @@ -371,19 +371,11 @@ export const CodeEditor: React.FC = ({ monaco.languages.registerCodeActionProvider(languageId, codeActions); } }); - - // Register themes - monaco.editor.defineTheme('euiColors', useDarkTheme ? DARK_THEME : LIGHT_THEME); - monaco.editor.defineTheme( - 'euiColorsTransparent', - useDarkTheme ? DARK_THEME_TRANSPARENT : LIGHT_THEME_TRANSPARENT - ); }, [ overrideEditorWillMount, editorWillMount, languageId, - useDarkTheme, suggestionProvider, signatureProvider, hoverProvider, @@ -479,16 +471,15 @@ export const CodeEditor: React.FC = ({ const { CopyButton } = useCopy({ isCopyable, value }); - useEffect(() => { - // Register themes when 'useDarkTheme' changes - monaco.editor.defineTheme('euiColors', useDarkTheme ? DARK_THEME : LIGHT_THEME); - monaco.editor.defineTheme( - 'euiColorsTransparent', - useDarkTheme ? DARK_THEME_TRANSPARENT : LIGHT_THEME_TRANSPARENT - ); - }, [useDarkTheme]); - - const theme = options?.theme ?? (transparentBackground ? 'euiColorsTransparent' : 'euiColors'); + const theme = + options?.theme ?? + (transparentBackground + ? useDarkTheme + ? CODE_EDITOR_DARK_THEME_TRANSPARENT_ID + : CODE_EDITOR_LIGHT_THEME_TRANSPARENT_ID + : useDarkTheme + ? CODE_EDITOR_DARK_THEME_ID + : CODE_EDITOR_LIGHT_THEME_ID); return (
', () => { httpRequestsMockHelpers.setLoadTelemetryResponse({}); httpRequestsMockHelpers.setLoadComponentTemplatesResponse([]); httpRequestsMockHelpers.setLoadTemplateResponse(templateToClone.name, templateToClone); + + // Mocking matchMedia to resolve TypeError: window.matchMedia is not a function + // For more info, see https://jestjs.io/docs/manual-mocks#mocking-methods-which-are-not-implemented-in-jsdom + Object.defineProperty(window, 'matchMedia', { + writable: true, + value: jest.fn().mockImplementation((query) => ({ + matches: false, + media: query, + onchange: null, + addListener: jest.fn(), // deprecated + removeListener: jest.fn(), // deprecated + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + dispatchEvent: jest.fn(), + })), + }); }); afterAll(() => { From 34c44496077e2fb58b4b1e81364c4311bddec8ae Mon Sep 17 00:00:00 2001 From: dkirchan <55240027+dkirchan@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:50:36 +0200 Subject: [PATCH 099/138] [Security] Quality Gate multi Organization for projects (#181027) ## Summary In order to run the tests concurrently we needed the ability to handle more than one organizations and rotation of api keys in order to create a project. This effort is covered by the job done for the cloud-handler (@elastic/security-engineering-productivity). The cloud-handler is a Python FastAPI service connected onto a Postgres Database, which handles the muiltiple organizations for the needs of the Security Kibana Quality Gate Testing - including the periodic pipeline and the future efforts to be able to run the tests from Devs against a real MKI. ## Description All the logic is pretty much handled in the `parallel_serverless.ts` script. [At this point](https://github.com/elastic/kibana/pull/181027/files#diff-a05c7d7d8448c53e20bbd60881deb4786bfffa3cdf654447732aed02e12b3867R223) we are getting the combination of PROXY_URL, PROXY_CLIENT_ID and PROXY_SECRET. All of these three should be defined as the first defines the URL of the proxy service and the latter define the authentication with the service. If all the three of the above mentioned variables are available, plus if the healthcheck for the service to be up and running is successful ([runs in this line](https://github.com/elastic/kibana/pull/181027/files#diff-a05c7d7d8448c53e20bbd60881deb4786bfffa3cdf654447732aed02e12b3867R255)) then the script starts creating environments through the proxy handler. Otherwise it goes back to the default single org execution (with the problems we have faced and tackling with this effort). If the flow procceeds with the proxy service then it creates the environment (the create environment request body is not changed at all so no change needs to be done in the test codebase) and then a response is returned indicating in the response body the organization-name that is being used. e.g.: ``` { "alias": "local-gizmo-tests-e2ebcd", "cloud_id": "local-gizmo-tests:ZXUtd2VzdC0xLmF3cy5xYS5lbGFzdGljLmNsb3VkJGUyZWJjZGZmMzY0YTRmYjliMjRmOGVkMGM0MjI2NThlLmVzJGUyZWJjZGZmMzY0YTRmYjliMjRmOGVkMGM0MjI2NThlLmti", "project_id": "e2ebcdff364a4fb9b24f8ed0c422658e", "name": "local-gizmo-tests", "region_id": "aws-eu-west-1", "project_type": "security", "admin_features_package": "standard", "creds_password": "f6RoNM84wQ4tBml3p13069uJ", "creds_username": "admin", "elasticsearch_endpoint": "https://local-gizmo-tests-e2ebcd.es.eu-west-1.aws.qa.elastic.cloud", "kibana_endpoint": "https://local-gizmo-tests-e2ebcd.kb.eu-west-1.aws.qa.elastic.cloud", "created_at": "2024-04-22T15:05:28.970745", "id": 1856, "organization_id": 16, **"organization_name": "sec-sol-auto-01"** } ``` Then this organization name is used to define the file with the roles which the saml authentication will be using in order to authenticate the users. This change is implemented in the following parts: - [The PROXY_ORG Cypress env var](https://github.com/elastic/kibana/pull/181027/files#diff-a05c7d7d8448c53e20bbd60881deb4786bfffa3cdf654447732aed02e12b3867R475) is defined. - [A roles filename is created](https://github.com/elastic/kibana/pull/181027/files#diff-5537ddd27eb2b8d7a4809e1bd9a28a4e6c23f3caa6a9b504b9c94ee037070315R34) if only the PROXY_ORG is defined and handed over to the SamlSessionManager. - [If the roles filename is provided,](https://github.com/elastic/kibana/pull/181027/files#diff-f63bfdabc35b838460de6b7e758d1bc168b54ba6ff418a8ad936d716c88af964R51) then it respects it, otherwise it uses the default `role_users.json` ## Relevant successful executions: - https://buildkite.com/elastic/security-serverless-quality-gate-kibana-periodic/builds/202 - https://buildkite.com/elastic/security-serverless-quality-gate-kibana-periodic/builds/203 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Gloria Hornero --- .../security_solution/api_integration.yml | 13 - .../api_integration_serverless_release.yml | 1 - ...mki_security_solution_defend_workflows.yml | 19 +- ...mki_security_solution_detection_engine.yml | 18 -- ...mki_security_solution_entity_analytics.yml | 17 -- .../mki_security_solution_explore.yml | 17 -- .../mki_security_solution_gen_ai.yml | 21 +- .../mki_security_solution_investigations.yml | 17 -- .../mki_security_solution_rule_management.yml | 18 -- .../api_integration/api-integration-tests.sh | 74 ++++-- .../prepare_vault_entries.sh | 18 ++ .../mki_security_solution_cypress.sh | 16 +- .../upload_image_metadata.sh | 5 +- packages/kbn-test/src/auth/session_manager.ts | 9 +- .../run_cypress/parallel_serverless.ts | 249 ++++-------------- .../project_handler/cloud_project_handler.ts | 170 ++++++++++++ .../project_handler/project_handler.ts | 85 ++++++ .../project_handler/proxy_project_handler.ts | 166 ++++++++++++ .../cypress/support/saml_auth.ts | 15 +- 19 files changed, 568 insertions(+), 380 deletions(-) create mode 100644 .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh create mode 100644 x-pack/plugins/security_solution/scripts/run_cypress/project_handler/cloud_project_handler.ts create mode 100644 x-pack/plugins/security_solution/scripts/run_cypress/project_handler/project_handler.ts create mode 100644 x-pack/plugins/security_solution/scripts/run_cypress/project_handler/proxy_project_handler.ts diff --git a/.buildkite/pipelines/security_solution/api_integration.yml b/.buildkite/pipelines/security_solution/api_integration.yml index 474fbc31b0bc3a..046d9fafe38731 100644 --- a/.buildkite/pipelines/security_solution/api_integration.yml +++ b/.buildkite/pipelines/security_solution/api_integration.yml @@ -1,19 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - queue: n2-16-spot - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: 'Upload runtime info' key: upload_runtime_info - depends_on: build_image agents: queue: n2-4-spot timeout_in_minutes: 300 @@ -24,7 +12,6 @@ steps: - group: 'Execute Tests' key: test_execution - depends_on: build_image steps: - label: Running exception_workflows:qa:serverless command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_workflows:qa:serverless diff --git a/.buildkite/pipelines/security_solution/api_integration_serverless_release.yml b/.buildkite/pipelines/security_solution/api_integration_serverless_release.yml index 25c6f0ce06b8ee..f1b9d002209d03 100644 --- a/.buildkite/pipelines/security_solution/api_integration_serverless_release.yml +++ b/.buildkite/pipelines/security_solution/api_integration_serverless_release.yml @@ -1,7 +1,6 @@ steps: - group: 'API Integration Serverless Release Tests' key: test_execution - depends_on: build_image steps: - label: Running integration tests for Serverless Exception Workflows command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_workflows:qa:serverless:release diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml index 11a99f29fbe146..974514f47c101b 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -31,6 +15,5 @@ steps: limit: 1 - command: "echo 'Running the defend worklows tests in this step" - depends_on: build_image key: test_defend_workflows - label: 'Serverless MKI QA Defend Workflows - Security Solution Cypress Tests' + label: "Serverless MKI QA Defend Workflows - Security Solution Cypress Tests" diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_detection_engine.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_detection_engine.yml index ff4eb22fb2e2df..a57659281d731c 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_detection_engine.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_detection_engine.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -33,7 +17,6 @@ steps: - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine label: 'Serverless MKI QA Detection Engine - Security Solution Cypress Tests' key: test_detection_engine - depends_on: build_image env: BK_TEST_SUITE_KEY: "serverless-cypress-detection-engine" agents: @@ -53,7 +36,6 @@ steps: - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions label: 'Serverless MKI QA Detection Engine - Exceptions - Security Solution Cypress Tests' key: test_detection_engine_exceptions - depends_on: build_image env: BK_TEST_SUITE_KEY: "serverless-cypress-detection-engine" agents: diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_entity_analytics.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_entity_analytics.yml index 518ec0268309b7..97947ed98fd24c 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_entity_analytics.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_entity_analytics.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -32,7 +16,6 @@ steps: - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:entity_analytics label: 'Serverless MKI QA Entity Analytics - Security Solution Cypress Tests' - depends_on: build_image key: test_entity_analytics env: BK_TEST_SUITE_KEY: "serverless-cypress-entity-analytics" diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_explore.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_explore.yml index a8ded33bfdd220..95a87db37bb31e 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_explore.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_explore.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -31,7 +15,6 @@ steps: limit: 1 - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore - depends_on: build_image key: test_explore label: 'Serverless MKI QA Explore - Security Solution Cypress Tests' env: diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_gen_ai.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_gen_ai.yml index 2b10150dd55ac6..90018caab7024f 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_gen_ai.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_gen_ai.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -31,8 +15,7 @@ steps: limit: 1 - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:ai_assistant - label: 'Serverless MKI QA AI Assistant - Security Solution Cypress Tests' - depends_on: build_image + label: "Serverless MKI QA AI Assistant - Security Solution Cypress Tests" key: test_ai_assistant env: BK_TEST_SUITE_KEY: "serverless-cypress-gen-ai" @@ -47,5 +30,5 @@ steps: parallelism: 1 retry: automatic: - - exit_status: '-1' + - exit_status: "-1" limit: 1 diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_investigations.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_investigations.yml index b7ddac79641ce6..39f4b66d5c6072 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_investigations.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_investigations.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -31,7 +15,6 @@ steps: limit: 1 - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations - depends_on: build_image key: test_investigations label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests' env: diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_rule_management.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_rule_management.yml index 58f10084bdcdaa..933d7b2de0ac50 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_rule_management.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_rule_management.yml @@ -1,23 +1,7 @@ steps: - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh - label: Build kibana image - key: build_image - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-qa - provider: gcp - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: "-1" - limit: 3 - - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh label: "Upload runtime info" key: upload_runtime_info - depends_on: build_image agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-qa @@ -32,7 +16,6 @@ steps: - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests' - depends_on: build_image key: test_rule_management env: BK_TEST_SUITE_KEY: "serverless-cypress-rule-management" @@ -52,7 +35,6 @@ steps: - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests' - depends_on: build_image key: test_rule_management_prebuilt_rules env: BK_TEST_SUITE_KEY: "serverless-cypress-rule-management" diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh index f87df9cb58baca..8b0ab8dcae33a1 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh @@ -10,57 +10,87 @@ source .buildkite/scripts/common/util.sh buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" -echo "--- Serverless Security Second Quality Gate" +source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh + cd x-pack/test/security_solution_api_integration set +e -QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key) -QA_CONSOLE_URL=$(vault_get security-solution-quality-gate qa_console_url) - # Generate a random 5-digit number random_number=$((10000 + $RANDOM % 90000)) -if [ -z "${KIBANA_MKI_USE_LATEST_COMMIT+x}" ] || [ "$KIBANA_MKI_USE_LATEST_COMMIT" = "0" ]; then - ENVIRONMENT_DETAILS=$(curl --location "$QA_CONSOLE_URL/api/v1/serverless/projects/security" \ - --header "Authorization: ApiKey $QA_API_KEY" \ +PROXY_URL="https://cloud-handler-test-r344edqiza-uc.a.run.app" +# Check the healthcheck of the proxy service +response=$(curl -s -o /dev/null -w "%{http_code}" "$PROXY_URL/healthcheck") +echo "Proxy Healthcheck Response code: $response" + +if [ "$response" -eq 200 ]; then + # Proxy service is up and running. Use the proxy to handle the projects. + CREATE_URL="$PROXY_URL/projects" + RESET_CREDS_URL="$PROXY_URL/projects/{project_id}/_reset-internal-credentials" + DELETE_URL="$PROXY_URL/projects/{project_id}" + AUTH="Basic $(vault_get security-solution-quality-gate-proxy base_64_encoded_auth)" +else + # Proxy service is not available. Use default single org execution mode using cloud QA directly. + CREATE_URL="$QA_CONSOLE_URL/api/v1/serverless/projects/security" + RESET_CREDS_URL="$QA_CONSOLE_URL/api/v1/serverless/projects/security/{project_id}/_reset-internal-credentials" + DELETE_URL="$QA_CONSOLE_URL/api/v1/serverless/projects/security/{project_id}" + AUTH="ApiKey $CLOUD_QA_API_KEY" +fi + + +if [ -z "${KIBANA_MKI_IMAGE_COMMIT+x}" ]; then + # There is no provided commit to be used so running against whatever image + # is already qualified in Cloud QA. + ENVIRONMENT_DETAILS=$(curl --location "$CREATE_URL" \ + --header "Authorization: $AUTH" \ --header 'Content-Type: application/json' \ --data '{ "name": "ftr-integration-tests-'$random_number'", "region_id": "aws-eu-west-1"}' | jq '.') else - KBN_COMMIT_HASH=${BUILDKITE_COMMIT:0:12} - ENVIRONMENT_DETAILS=$(curl --location "$QA_CONSOLE_URL/api/v1/serverless/projects/security" \ - --header "Authorization: ApiKey $QA_API_KEY" \ + # A commit is provided so it will be used to run the tests against this qualified image. + KBN_COMMIT_HASH=${KIBANA_MKI_IMAGE_COMMIT:0:12} + ENVIRONMENT_DETAILS=$(curl --location "$CREATE_URL" \ + --header "Authorization: $AUTH" \ --header 'Content-Type: application/json' \ --data '{ "name": "ftr-integration-tests-'$random_number'", "region_id": "aws-eu-west-1", "overrides": { "kibana": { - "docker_image" : "docker.elastic.co/kibana-ci/kibana-serverless:sec-sol-qg-'$KBN_COMMIT_HASH'" + "docker_image" : "docker.elastic.co/kibana-ci/kibana-serverless:git-'$KBN_COMMIT_HASH'" } } }' | jq '.') fi +if [ "$response" -eq 200 ]; then + # Proxy is up and running so reading the ES and KB endpoints from the proxy response. + ES_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.elasticsearch_endpoint') + KB_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.kibana_endpoint') + ID=$(echo $ENVIRONMENT_DETAILS | jq -r '.project_id') +else + # Proxy is unavailable so reading the ES and KB endpoints from the cloud QA response. + ES_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.endpoints.elasticsearch') + KB_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.endpoints.kibana') + ID=$(echo $ENVIRONMENT_DETAILS | jq -r '.id') +fi NAME=$(echo $ENVIRONMENT_DETAILS | jq -r '.name') -ID=$(echo $ENVIRONMENT_DETAILS | jq -r '.id') -ES_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.endpoints.elasticsearch') -KB_URL=$(echo $ENVIRONMENT_DETAILS | jq -r '.endpoints.kibana') # Wait five seconds for the project to appear sleep 5 # Resetting the credentials of the elastic user in the project -CREDS_BODY=$(curl -s --location --request POST "$QA_CONSOLE_URL/api/v1/serverless/projects/security/$ID/_reset-internal-credentials" \ - --header "Authorization: ApiKey $QA_API_KEY" \ +RESET_CREDENTIALS_URL=$(echo "$RESET_CREDS_URL" | sed "s/{project_id}/$ID/g") +CREDS_BODY=$(curl -s --location --request POST "$RESET_CREDENTIALS_URL" \ + --header "Authorization: $AUTH" \ --header 'Content-Type: application/json' | jq '.') USERNAME=$(echo $CREDS_BODY | jq -r '.username') PASSWORD=$(echo $CREDS_BODY | jq -r '.password') -AUTH=$(echo "$USERNAME:$PASSWORD") +PROJECT_AUTH=$(echo "$USERNAME:$PASSWORD") # Checking if Elasticsearch has status green while : ; do - STATUS=$(curl -u $AUTH --location "$ES_URL:443/_cluster/health?wait_for_status=green&timeout=50s" | jq -r '.status') + STATUS=$(curl -u $PROJECT_AUTH --location "$ES_URL:443/_cluster/health?wait_for_status=green&timeout=50s" | jq -r '.status') if [ "$STATUS" != "green" ]; then echo "Sleeping for 40s to wait for ES status to be green..." sleep 40 @@ -72,7 +102,7 @@ done # Checking if Kibana is available while : ; do - STATUS=$(curl -u $AUTH --location "$KB_URL:443/api/status" | jq -r '.status.overall.level') + STATUS=$(curl -u $PROJECT_AUTH --location "$KB_URL:443/api/status" | jq -r '.status.overall.level') if [ "$STATUS" != "available" ]; then echo "Sleeping for 15s to wait for Kibana to be available..." sleep 15 @@ -90,11 +120,13 @@ FORMATTED_KB_URL="${KB_URL/https:\/\//}" # This is used in order to wait for the environment to be ready. sleep 150 +echo "--- Triggering API tests for $1" TEST_CLOUD=1 TEST_ES_URL="https://$USERNAME:$PASSWORD@$FORMATTED_ES_URL:443" TEST_KIBANA_URL="https://$USERNAME:$PASSWORD@$FORMATTED_KB_URL:443" yarn run $1 cmd_status=$? echo "Exit code with status: $cmd_status" -curl --location --request DELETE "$QA_CONSOLE_URL/api/v1/serverless/projects/security/$ID" \ - --header "Authorization: ApiKey $QA_API_KEY" +DELETE_PROJECT_URL=$(echo "$DELETE_URL" | sed "s/{project_id}/$ID/g") +curl --location --request DELETE "$DELETE_PROJECT_URL" \ + --header "Authorization: $AUTH" exit $cmd_status diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh new file mode 100644 index 00000000000000..85f361752f6f64 --- /dev/null +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +mkdir .ftr + +# The role-users file that is used as a fallback if the proxy service is unavailable. +vault_get security-quality-gate/role-users data -format=json > .ftr/role_users.json +# The role-users files relevant to the proxy service and its orgs. +vault_get security-quality-gate/role-users/sec-sol-auto-01 data -format=json > .ftr/sec-sol-auto-01.json +vault_get security-quality-gate/role-users/sec-sol-auto-02 data -format=json > .ftr/sec-sol-auto-02.json +vault_get security-quality-gate/role-users/sec-sol-auto-03 data -format=json > .ftr/sec-sol-auto-03.json + +# The vault entries relevant to QA Cloud +export CLOUD_QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key) +export QA_CONSOLE_URL=$(vault_get security-solution-quality-gate qa_console_url) +# The vault entries relevant to the Proxy service (Cloud Handler) +export PROXY_URL=$(vault_get security-solution-quality-gate-proxy proxy_url_test) +export PROXY_CLIENT_ID=$(vault_get security-solution-quality-gate-proxy client_id) +export PROXY_SECRET=$(vault_get security-solution-quality-gate-proxy secret) diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh index cf0dc230b0f404..6507f017504247 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh @@ -16,20 +16,12 @@ export JOB=kibana-security-solution-chrome buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" -mkdir .ftr -vault_get security-quality-gate/role-users data -format=json > .ftr/role_users.json +source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh cd x-pack/test/security_solution_cypress set +e -if [ -z "${KIBANA_MKI_USE_LATEST_COMMIT+x}" ] || [ "$KIBANA_MKI_USE_LATEST_COMMIT" = "0" ]; then - KIBANA_OVERRIDE_FLAG=0 -else - KIBANA_OVERRIDE_FLAG=1 -fi - -QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key) -QA_CONSOLE_URL=$(vault_get security-solution-quality-gate qa_console_url) -BK_ANALYTICS_API_KEY=$(vault_get security-solution-quality-gate $BK_TEST_SUITE_KEY) +export BK_ANALYTICS_API_KEY=$(vault_get security-solution-quality-gate $BK_TEST_SUITE_KEY) -QA_CONSOLE_URL=$QA_CONSOLE_URL KIBANA_MKI_USE_LATEST_COMMIT=$KIBANA_OVERRIDE_FLAG BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status +echo "--- Triggering Kibana tests for $1" +BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh index c1a22d221cafc7..a39c51c07a47c0 100644 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh @@ -2,11 +2,10 @@ echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-serverless" -KIBANA_CURRENT_COMMIT=${KIBANA_BASE_IMAGE}:sec-sol-qg-${BUILDKITE_COMMIT:0:12} KIBANA_LATEST=${KIBANA_BASE_IMAGE}:latest -if [ "$KIBANA_MKI_USE_LATEST_COMMIT" = "1" ]; then - KBN_IMAGE=${KIBANA_CURRENT_COMMIT} +if [ -v KIBANA_MKI_IMAGE_COMMIT ]; then + KBN_IMAGE=${KIBANA_BASE_IMAGE}:git-${KIBANA_MKI_IMAGE_COMMIT:0:12} else KBN_IMAGE=${KIBANA_LATEST} fi diff --git a/packages/kbn-test/src/auth/session_manager.ts b/packages/kbn-test/src/auth/session_manager.ts index 7e7b862aa84b2f..a8edca981c1d4d 100644 --- a/packages/kbn-test/src/auth/session_manager.ts +++ b/packages/kbn-test/src/auth/session_manager.ts @@ -35,18 +35,23 @@ export interface SamlSessionManagerOptions { * Manages cookies associated with user roles */ export class SamlSessionManager { + private readonly DEFAULT_ROLES_FILE_NAME: string = 'role_users.json'; private readonly isCloud: boolean; private readonly kbnHost: string; private readonly kbnClient: KbnClient; private readonly log: ToolingLog; private readonly roleToUserMap: Map; private readonly sessionCache: Map; - private readonly userRoleFilePath = resolve(REPO_ROOT, '.ftr', 'role_users.json'); private readonly supportedRoles: string[]; + private readonly userRoleFilePath: string; - constructor(options: SamlSessionManagerOptions) { + constructor(options: SamlSessionManagerOptions, rolesFilename?: string) { this.isCloud = options.isCloud; this.log = options.log; + // if the rolesFilename is provided, respect it. Otherwise use DEFAULT_ROLES_FILE_NAME. + const rolesFile = rolesFilename ? rolesFilename : this.DEFAULT_ROLES_FILE_NAME; + this.log.info(`Using the file ${rolesFile} for the role users`); + this.userRoleFilePath = resolve(REPO_ROOT, '.ftr', rolesFile); const hostOptionsWithoutAuth = { protocol: options.hostOptions.protocol, hostname: options.hostOptions.hostname, diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 2c5ca1e049fae1..8c3f13eefffce1 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -28,42 +28,9 @@ import { exec } from 'child_process'; import { renderSummaryTable } from './print_run'; import { parseTestFileConfig, retrieveIntegrations } from './utils'; -interface ProductType { - product_line: string; - product_tier: string; -} - -interface OverrideEntry { - docker_image: string; -} - -interface ProductOverrides { - kibana?: OverrideEntry; - elasticsearch?: OverrideEntry; - fleet?: OverrideEntry; - cluster?: OverrideEntry; -} - -interface CreateProjectRequestBody { - name: string; - region_id: string; - product_types?: ProductType[]; - overrides?: ProductOverrides; -} - -interface Project { - name: string; - id: string; - region: string; - es_url: string; - kb_url: string; - product: string; -} - -interface Credentials { - username: string; - password: string; -} +import type { ProductType, Credentials, ProjectHandler } from './project_handler/project_handler'; +import { CloudHandler } from './project_handler/cloud_project_handler'; +import { ProxyHandler } from './project_handler/proxy_project_handler'; const DEFAULT_CONFIGURATION: Readonly = [ { product_line: 'security', product_tier: 'complete' }, @@ -71,7 +38,6 @@ const DEFAULT_CONFIGURATION: Readonly = [ { product_line: 'endpoint', product_tier: 'complete' }, ] as const; -const DEFAULT_REGION = 'aws-eu-west-1'; const PROJECT_NAME_PREFIX = 'kibana-cypress-security-solution-ephemeral'; const BASE_ENV_URL = `${process.env.QA_CONSOLE_URL}`; let log: ToolingLog; @@ -96,156 +62,27 @@ const getApiKeyFromElasticCloudJsonFile = (): string | undefined => { } }; -// Method to invoke the create project API for serverless. -async function createSecurityProject( - projectName: string, - apiKey: string, - productTypes: ProductType[], - commit: string -): Promise { - const body: CreateProjectRequestBody = { - name: projectName, - region_id: DEFAULT_REGION, - product_types: productTypes, - }; - - log.info(`Kibana override flag equals to ${process.env.KIBANA_MKI_USE_LATEST_COMMIT}!`); - if ( - (process.env.KIBANA_MKI_USE_LATEST_COMMIT && - process.env.KIBANA_MKI_USE_LATEST_COMMIT === '1') || - commit - ) { - const override = commit ? commit : process.env.BUILDKITE_COMMIT; - const kibanaOverrideImage = `${override?.substring(0, 12)}`; - log.info( - `Overriding Kibana image in the MKI with docker.elastic.co/kibana-ci/kibana-serverless:sec-sol-qg-${kibanaOverrideImage}` - ); - body.overrides = { - kibana: { - docker_image: `docker.elastic.co/kibana-ci/kibana-serverless:sec-sol-qg-${kibanaOverrideImage}`, - }, - }; - } - - try { - const response = await axios.post(`${BASE_ENV_URL}/api/v1/serverless/projects/security`, body, { - headers: { - Authorization: `ApiKey ${apiKey}`, - }, - }); - return { - name: response.data.name, - id: response.data.id, - region: response.data.region_id, - es_url: `${response.data.endpoints.elasticsearch}:443`, - kb_url: `${response.data.endpoints.kibana}:443`, - product: response.data.type, - }; - } catch (error) { - if (error instanceof AxiosError) { - const errorData = JSON.stringify(error.response?.data); - log.error(`${error.response?.status}:${errorData}`); - } else { - log.error(`${error.message}`); - } - } -} - -// Method to invoke the delete project API for serverless. -async function deleteSecurityProject( - projectId: string, - projectName: string, - apiKey: string -): Promise { - try { - await axios.delete(`${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}`, { - headers: { - Authorization: `ApiKey ${apiKey}`, - }, - }); - log.info(`Project ${projectName} was successfully deleted!`); - } catch (error) { - if (error instanceof AxiosError) { - log.error(`${error.response?.status}:${error.response?.data}`); - } else { - log.error(`${error.message}`); - } - } -} +// Check if proxy service is up and running executing a healthcheck call. +function proxyHealthcheck(proxyUrl: string): Promise { + const fetchHealthcheck = async (attemptNum: number) => { + log.info(`Retry number ${attemptNum} to check if Elasticsearch is green.`); -// Method to reset the credentials for the created project. -async function resetCredentials( - projectId: string, - runnerId: string, - apiKey: string -): Promise { - log.info(`${runnerId} : Reseting credentials`); - - const fetchResetCredentialsStatusAttempt = async (attemptNum: number) => { - const response = await axios.post( - `${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}/_reset-internal-credentials`, - {}, - { - headers: { - Authorization: `ApiKey ${apiKey}`, - }, - } - ); - log.info('Credentials have ben reset'); - return { - password: response.data.password, - username: response.data.username, - }; + const response = await axios.get(`${proxyUrl}/healthcheck`); + log.info(`The proxy service is available.`); + return response.status === 200; }; - const retryOptions = { onFailedAttempt: (error: Error | AxiosError) => { - if (error instanceof AxiosError && error.code === 'ENOTFOUND') { - log.info('Project is not reachable. A retry will be triggered soon..'); - } else { - log.error(`${error.message}`); + if (error instanceof AxiosError) { + log.info(`The proxy service is not available. A retry will be triggered soon...`); } }, - retries: 100, + retries: 4, factor: 2, maxTimeout: 20000, }; - return pRetry(fetchResetCredentialsStatusAttempt, retryOptions); -} - -// Wait until Project is initialized -function waitForProjectInitialized(projectId: string, apiKey: string): Promise { - const fetchProjectStatusAttempt = async (attemptNum: number) => { - log.info(`Retry number ${attemptNum} to check if project is initialized.`); - const response = await axios.get( - `${BASE_ENV_URL}/api/v1/serverless/projects/security/${projectId}/status`, - { - headers: { - Authorization: `ApiKey ${apiKey}`, - }, - } - ); - if (response.data.phase !== 'initialized') { - log.info(response.data); - throw new Error('Project is not initialized. A retry will be triggered soon...'); - } else { - log.info('Project is initialized'); - } - }; - const retryOptions = { - onFailedAttempt: (error: Error | AxiosError) => { - if (error instanceof AxiosError && error.code === 'ENOTFOUND') { - log.info('Project is not reachable. A retry will be triggered soon...'); - } else { - log.error(`${error.message}`); - } - }, - retries: 100, - factor: 2, - maxTimeout: 20000, - }; - return pRetry(fetchProjectStatusAttempt, retryOptions); + return pRetry(fetchHealthcheck, retryOptions); } // Wait until elasticsearch status goes green @@ -406,10 +243,25 @@ export const cli = () => { return process.exit(1); } + const PROXY_URL = process.env.PROXY_URL ? process.env.PROXY_URL : undefined; + const PROXY_SECRET = process.env.PROXY_SECRET ? process.env.PROXY_SECRET : undefined; + const PROXY_CLIENT_ID = process.env.PROXY_CLIENT_ID ? process.env.PROXY_CLIENT_ID : undefined; + const API_KEY = process.env.CLOUD_QA_API_KEY ? process.env.CLOUD_QA_API_KEY : getApiKeyFromElasticCloudJsonFile(); + let cloudHandler: ProjectHandler; + if (PROXY_URL && PROXY_CLIENT_ID && PROXY_SECRET && (await proxyHealthcheck(PROXY_URL))) { + cloudHandler = new ProxyHandler(PROXY_URL, PROXY_CLIENT_ID, PROXY_SECRET); + } else if (API_KEY) { + cloudHandler = new CloudHandler(API_KEY, BASE_ENV_URL); + } else { + log.info('PROXY_URL or API KEY which are needed to create project could not be retrieved.'); + // eslint-disable-next-line no-process-exit + return process.exit(1); + } + const PARALLEL_COUNT = process.env.PARALLEL_COUNT ? Number(process.env.PARALLEL_COUNT) : 1; if (!process.env.CLOUD_ENV) { @@ -417,7 +269,6 @@ export const cli = () => { 'The cloud environment to be provided with the env var CLOUD_ENV. Currently working only for QA so the script can proceed.' ); // Abort when more environments will be integrated - // return process.exit(0); } @@ -469,15 +320,16 @@ ${JSON.stringify(argv, null, 2)} const isOpen = argv._.includes('open'); const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); - // KIBANA_MKI_USE_LATEST_COMMIT === 1 means that we are overriding the image for the periodic pipeline execution. - // We don't override the image when executing the tests on the second quality gate. - if ( - !process.env.KIBANA_MKI_USE_LATEST_COMMIT || - process.env.KIBANA_MKI_USE_LATEST_COMMIT !== '1' - ) { - cypressConfigFile.env.grepTags = - '@serverlessQA --@skipInServerless --@skipInServerlessMKI '; + + // if KIBANA_MKI_QUALITY_GATE exists and has a value, it means that we are running the tests against the second + // quality gate. + if (process.env.KIBANA_MKI_QUALITY_GATE) { + log.info( + 'KIBANA_MKI_QUALITY_GATE is provided, so @serverlessQA --@skipInServerless --@skipInServerlessMKI tags will run.' + ); + cypressConfigFile.env.grepTags = '@serverlessQA --@skipInServerless --@skipInServerlessMKI'; } + const tier: string = argv.tier; const endpointAddon: boolean = argv.endpointAddon; const cloudAddon: boolean = argv.cloudAddon; @@ -559,17 +411,10 @@ ${JSON.stringify(cypressConfigFile, null, 2)} ? getProductTypes(tier, endpointAddon, cloudAddon) : (parseTestFileConfig(filePath).productTypes as ProductType[]); - if (!API_KEY) { - log.info('API KEY to create project could not be retrieved.'); - // eslint-disable-next-line no-process-exit - return process.exit(1); - } - log.info(`${id}: Creating project ${PROJECT_NAME}...`); // Creating project for the test to run - const project = await createSecurityProject( + const project = await cloudHandler.createSecurityProject( PROJECT_NAME, - API_KEY, productTypes, commit ); @@ -581,12 +426,15 @@ ${JSON.stringify(cypressConfigFile, null, 2)} } context.addCleanupTask(() => { - const command = `curl -X DELETE ${BASE_ENV_URL}/api/v1/serverless/projects/security/${project.id} -H "Authorization: ApiKey ${API_KEY}"`; - exec(command); + let command: string; + if (cloudHandler instanceof CloudHandler) { + command = `curl -X DELETE ${BASE_ENV_URL}/api/v1/serverless/projects/security/${project.id} -H "Authorization: ApiKey ${API_KEY}"`; + exec(command); + } }); // Reset credentials for elastic user - const credentials = await resetCredentials(project.id, id, API_KEY); + const credentials = await cloudHandler.resetCredentials(project.id, id); if (!credentials) { log.info('Credentials could not be reset.'); @@ -595,7 +443,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} } // Wait for project to be initialized - await waitForProjectInitialized(project.id, API_KEY); + await cloudHandler.waitForProjectInitialized(project.id); // Base64 encode the credentials in order to invoke ES and KB APIs const auth = btoa(`${credentials.username}:${credentials.password}`); @@ -620,6 +468,9 @@ ${JSON.stringify(cypressConfigFile, null, 2)} ELASTICSEARCH_USERNAME: credentials.username, ELASTICSEARCH_PASSWORD: credentials.password, + // Used in order to handle the correct role_users file loading. + PROXY_ORG: PROXY_URL ? project.proxy_org_name : undefined, + KIBANA_URL: project.kb_url, KIBANA_USERNAME: credentials.username, KIBANA_PASSWORD: credentials.password, @@ -674,7 +525,7 @@ ${JSON.stringify(cypressConfigFile, null, 2)} } // Delete serverless project log.info(`${id} : Deleting project ${PROJECT_NAME}...`); - await deleteSecurityProject(project.id, PROJECT_NAME, API_KEY); + await cloudHandler.deleteSecurityProject(project.id, PROJECT_NAME); } catch (error) { // False positive // eslint-disable-next-line require-atomic-updates diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/cloud_project_handler.ts b/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/cloud_project_handler.ts new file mode 100644 index 00000000000000..89166701ee2067 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/cloud_project_handler.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import axios, { AxiosError } from 'axios'; +import pRetry from 'p-retry'; +import type { + ProductType, + Project, + CreateProjectRequestBody, + Credentials, +} from './project_handler'; +import { ProjectHandler } from './project_handler'; + +const DEFAULT_REGION = 'aws-eu-west-1'; + +export class CloudHandler extends ProjectHandler { + apiKey: string; + + constructor(apiKey: string, baseEnvUrl: string) { + super(baseEnvUrl); + this.apiKey = apiKey; + } + + // Method to invoke the create project API for serverless. + async createSecurityProject( + projectName: string, + productTypes: ProductType[], + commit: string + ): Promise { + const body: CreateProjectRequestBody = { + name: projectName, + region_id: DEFAULT_REGION, + product_types: productTypes, + }; + + if (process.env.KIBANA_MKI_IMAGE_COMMIT || commit) { + const override = commit ? commit : process.env.KIBANA_MKI_IMAGE_COMMIT; + const kibanaOverrideImage = `${override?.substring(0, 12)}`; + this.log.info(`Kibana Image Commit under test: ${process.env.KIBANA_MKI_IMAGE_COMMIT}!`); + this.log.info( + `Overriding Kibana image in the MKI with docker.elastic.co/kibana-ci/kibana-serverless:git-${kibanaOverrideImage}` + ); + body.overrides = { + kibana: { + docker_image: `docker.elastic.co/kibana-ci/kibana-serverless:git-${kibanaOverrideImage}`, + }, + }; + } + + try { + const response = await axios.post( + `${this.baseEnvUrl}/api/v1/serverless/projects/security`, + body, + { + headers: { + Authorization: `ApiKey ${this.apiKey}`, + }, + } + ); + return { + name: response.data.name, + id: response.data.id, + region: response.data.region_id, + es_url: `${response.data.endpoints.elasticsearch}:443`, + kb_url: `${response.data.endpoints.kibana}:443`, + product: response.data.type, + }; + } catch (error) { + if (error instanceof AxiosError) { + const errorData = JSON.stringify(error.response?.data); + this.log.error(`${error.response?.status}:${errorData}`); + } else { + this.log.error(`${error.message}`); + } + } + } + + // Method to invoke the delete project API for serverless. + async deleteSecurityProject(projectId: string, projectName: string): Promise { + try { + await axios.delete(`${this.baseEnvUrl}/api/v1/serverless/projects/security/${projectId}`, { + headers: { + Authorization: `ApiKey ${this.apiKey}`, + }, + }); + this.log.info(`Project ${projectName} was successfully deleted!`); + } catch (error) { + if (error instanceof AxiosError) { + this.log.error(`${error.response?.status}:${error.response?.data}`); + } else { + this.log.error(`${error.message}`); + } + } + } + + // Method to reset the credentials for the created project. + resetCredentials(projectId: string, runnerId: string): Promise { + this.log.info(`${runnerId} : Reseting credentials`); + + const fetchResetCredentialsStatusAttempt = async (attemptNum: number) => { + const response = await axios.post( + `${this.baseEnvUrl}/api/v1/serverless/projects/security/${projectId}/_reset-internal-credentials`, + {}, + { + headers: { + Authorization: `ApiKey ${this.apiKey}`, + }, + } + ); + this.log.info('Credentials have ben reset'); + return { + password: response.data.password, + username: response.data.username, + }; + }; + + const retryOptions = { + onFailedAttempt: (error: Error | AxiosError) => { + if (error instanceof AxiosError && error.code === 'ENOTFOUND') { + this.log.info('Project is not reachable. A retry will be triggered soon..'); + } else { + this.log.error(`${error.message}`); + } + }, + retries: 100, + factor: 2, + maxTimeout: 20000, + }; + + return pRetry(fetchResetCredentialsStatusAttempt, retryOptions); + } + + // Wait until Project is initialized + waitForProjectInitialized(projectId: string): Promise { + const fetchProjectStatusAttempt = async (attemptNum: number) => { + this.log.info(`Retry number ${attemptNum} to check if project is initialized.`); + const response = await axios.get( + `${this.baseEnvUrl}/api/v1/serverless/projects/security/${projectId}/status`, + { + headers: { + Authorization: `ApiKey ${this.apiKey}`, + }, + } + ); + if (response.data.phase !== 'initialized') { + this.log.info(response.data); + throw new Error('Project is not initialized. A retry will be triggered soon...'); + } else { + this.log.info('Project is initialized'); + } + }; + const retryOptions = { + onFailedAttempt: (error: Error | AxiosError) => { + if (error instanceof AxiosError && error.code === 'ENOTFOUND') { + this.log.info('Project is not reachable. A retry will be triggered soon...'); + } else { + this.log.error(`${error.message}`); + } + }, + retries: 100, + factor: 2, + maxTimeout: 20000, + }; + return pRetry(fetchProjectStatusAttempt, retryOptions); + } +} diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/project_handler.ts b/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/project_handler.ts new file mode 100644 index 00000000000000..199df9c4fb4c01 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/project_handler.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; + +export interface ProductType { + product_line: string; + product_tier: string; +} + +export interface OverrideEntry { + docker_image: string; +} + +export interface ProductOverrides { + kibana?: OverrideEntry; + elasticsearch?: OverrideEntry; + fleet?: OverrideEntry; + cluster?: OverrideEntry; +} + +export interface CreateProjectRequestBody { + name: string; + region_id: string; + product_types?: ProductType[]; + overrides?: ProductOverrides; +} + +export interface Project { + name: string; + id: string; + region: string; + es_url: string; + kb_url: string; + product: string; + proxy_id?: number; + proxy_org_id?: number; + proxy_org_name?: string; +} + +export interface Credentials { + username: string; + password: string; +} + +export class ProjectHandler { + private readonly DEFAULT_ERROR_MSG: string = + 'The method needs to be overriden when the class is inherited!'; + + baseEnvUrl: string; + log: ToolingLog; + + constructor(baseEnvUrl: string) { + this.baseEnvUrl = baseEnvUrl; + this.log = new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }); + } + + // Method to invoke the create project API for serverless. + async createSecurityProject( + projectName: string, + productTypes: ProductType[], + commit: string + ): Promise { + throw new Error(this.DEFAULT_ERROR_MSG); + } + + async deleteSecurityProject(projectId: string, projectName: string): Promise { + throw new Error(this.DEFAULT_ERROR_MSG); + } + + resetCredentials(projectId: string, runnerId: string): Promise { + throw new Error(this.DEFAULT_ERROR_MSG); + } + + waitForProjectInitialized(projectId: string): Promise { + throw new Error(this.DEFAULT_ERROR_MSG); + } +} diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/proxy_project_handler.ts b/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/proxy_project_handler.ts new file mode 100644 index 00000000000000..08e5b418b83ac6 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/run_cypress/project_handler/proxy_project_handler.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import axios, { AxiosError } from 'axios'; +import pRetry from 'p-retry'; +import type { + ProductType, + Project, + CreateProjectRequestBody, + Credentials, +} from './project_handler'; +import { ProjectHandler } from './project_handler'; + +const DEFAULT_REGION = 'aws-eu-west-1'; + +export class ProxyHandler extends ProjectHandler { + proxyAuth: string; + + constructor(baseEnvUrl: string, proxyClientId: string, proxySecret: string) { + super(baseEnvUrl); + this.proxyAuth = btoa(`${proxyClientId}:${proxySecret}`); + } + + // Method to invoke the create project API for serverless. + async createSecurityProject( + projectName: string, + productTypes: ProductType[], + commit: string + ): Promise { + const body: CreateProjectRequestBody = { + name: projectName, + region_id: DEFAULT_REGION, + product_types: productTypes, + }; + + if (process.env.KIBANA_MKI_IMAGE_COMMIT || commit) { + const override = commit ? commit : process.env.KIBANA_MKI_IMAGE_COMMIT; + const kibanaOverrideImage = `${override?.substring(0, 12)}`; + this.log.info(`Kibana Image Commit under test: ${process.env.KIBANA_MKI_IMAGE_COMMIT}!`); + this.log.info( + `Overriding Kibana image in the MKI with docker.elastic.co/kibana-ci/kibana-serverless:git-${kibanaOverrideImage}` + ); + body.overrides = { + kibana: { + docker_image: `docker.elastic.co/kibana-ci/kibana-serverless:git-${kibanaOverrideImage}`, + }, + }; + } + + try { + const response = await axios.post(`${this.baseEnvUrl}/projects`, body, { + headers: { + Authorization: `Basic ${this.proxyAuth}`, + }, + }); + return { + name: response.data.name, + id: response.data.project_id, + region: response.data.region_id, + es_url: `${response.data.elasticsearch_endpoint}:443`, + kb_url: `${response.data.kibana_endpoint}:443`, + product: response.data.project_type, + proxy_id: response.data.id, + proxy_org_id: response.data.organization_id, + proxy_org_name: response.data.organization_name, + }; + } catch (error) { + if (error instanceof AxiosError) { + const errorData = JSON.stringify(error.response?.data); + this.log.error(`${error.response?.status}:${errorData}`); + } else { + this.log.error(`${error.message}`); + } + } + } + + // Method to invoke the delete project API for serverless. + async deleteSecurityProject(projectId: string, projectName: string): Promise { + try { + await axios.delete(`${this.baseEnvUrl}/projects/${projectId}`, { + headers: { + Authorization: `Basic ${this.proxyAuth}`, + }, + }); + this.log.info(`Project ${projectName} was successfully deleted!`); + } catch (error) { + if (error instanceof AxiosError) { + this.log.error(`${error.response?.status}:${error.response?.data}`); + } else { + this.log.error(`${error.message}`); + } + } + } + + // Method to reset the credentials for the created project. + resetCredentials(projectId: string, runnerId: string): Promise { + this.log.info(`${runnerId} : Reseting credentials`); + + const fetchResetCredentialsStatusAttempt = async (attemptNum: number) => { + const response = await axios.post( + `${this.baseEnvUrl}/projects/${projectId}/_reset-internal-credentials`, + {}, + { + headers: { + Authorization: `Basic ${this.proxyAuth}`, + }, + } + ); + this.log.info('Credentials have ben reset'); + return { + password: response.data.password, + username: response.data.username, + }; + }; + + const retryOptions = { + onFailedAttempt: (error: Error | AxiosError) => { + if (error instanceof AxiosError && error.code === 'ENOTFOUND') { + this.log.info('Project is not reachable. A retry will be triggered soon..'); + } else { + this.log.error(`${error.message}`); + } + }, + retries: 100, + factor: 2, + maxTimeout: 20000, + }; + + return pRetry(fetchResetCredentialsStatusAttempt, retryOptions); + } + + // Wait until Project is initialized + waitForProjectInitialized(projectId: string): Promise { + const fetchProjectStatusAttempt = async (attemptNum: number) => { + this.log.info(`Retry number ${attemptNum} to check if project is initialized.`); + const response = await axios.get(`${this.baseEnvUrl}/projects/${projectId}/status`, { + headers: { + Authorization: `Basic ${this.proxyAuth}`, + }, + }); + if (response.data.phase !== 'initialized') { + this.log.info(response.data); + throw new Error('Project is not initialized. A retry will be triggered soon...'); + } else { + this.log.info('Project is initialized'); + } + }; + const retryOptions = { + onFailedAttempt: (error: Error | AxiosError) => { + if (error instanceof AxiosError && error.code === 'ENOTFOUND') { + this.log.info('Project is not reachable. A retry will be triggered soon...'); + } else { + this.log.error(`${error.message}`); + } + }, + retries: 100, + factor: 2, + maxTimeout: 20000, + }; + return pRetry(fetchProjectStatusAttempt, retryOptions); + } +} diff --git a/x-pack/test/security_solution_cypress/cypress/support/saml_auth.ts b/x-pack/test/security_solution_cypress/cypress/support/saml_auth.ts index 0026f6c91ec27f..4c26b46a0f62a7 100644 --- a/x-pack/test/security_solution_cypress/cypress/support/saml_auth.ts +++ b/x-pack/test/security_solution_cypress/cypress/support/saml_auth.ts @@ -30,11 +30,16 @@ export const samlAuthentication = async ( on('task', { getSessionCookie: async (role: string | SecurityRoleName): Promise => { - const sessionManager = new SamlSessionManager({ - hostOptions, - log, - isCloud: config.env.CLOUD_SERVERLESS, - }); + // If config.env.PROXY_ORG is set, it means that proxy service is used to create projects. Define the proxy org filename to override the roles. + const rolesFilename = config.env.PROXY_ORG ? `${config.env.PROXY_ORG}.json` : undefined; + const sessionManager = new SamlSessionManager( + { + hostOptions, + log, + isCloud: config.env.CLOUD_SERVERLESS, + }, + rolesFilename + ); return sessionManager.getSessionCookieForRole(role); }, }); From 00d601a7ad8c300fe9b2bea4c8ce29178b7637b9 Mon Sep 17 00:00:00 2001 From: Mykola Harmash Date: Thu, 25 Apr 2024 15:00:40 +0200 Subject: [PATCH 100/138] [Onboarding] Add virtual Upload File card to the search results (#181690) Closes https://github.com/elastic/kibana/issues/181679 * Adds the Upload File card * Disables the Firehose card * Fixes a bug when app was crashing when invalid syntax is entered in the search field --- .../onboarding_flow_form.tsx | 12 +++-- .../use_virtual_search_results.ts | 46 +++++++++++++++++++ .../application/packages_list/index.tsx | 8 ++-- 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/use_virtual_search_results.ts diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/onboarding_flow_form.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/onboarding_flow_form.tsx index 8714917f21a321..197435a396432f 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/onboarding_flow_form.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/onboarding_flow_form.tsx @@ -26,6 +26,7 @@ import { OnboardingFlowPackageList } from '../packages_list'; import { useCustomMargin } from '../shared/use_custom_margin'; import { Category } from './types'; import { useCustomCardsForCategory } from './use_custom_cards_for_category'; +import { useVirtualSearchResults } from './use_virtual_search_results'; interface UseCaseOption { id: Category; @@ -140,6 +141,7 @@ export const OnboardingFlowForm: FunctionComponent = () => { createCollectionCardHandler, searchParams.get('category') as Category | null ); + const virtualSearchResults = useVirtualSearchResults(); return ( @@ -216,10 +218,12 @@ export const OnboardingFlowForm: FunctionComponent = () => { setSearchQuery={setIntegrationSearch} flowCategory={searchParams.get('category')} ref={packageListRef} - customCards={customCards?.filter( - // Filter out collection cards and regular integrations that show up via search anyway - (card) => card.type === 'virtual' && !card.isCollectionCard - )} + customCards={customCards + ?.filter( + // Filter out collection cards and regular integrations that show up via search anyway + (card) => card.type === 'virtual' && !card.isCollectionCard + ) + .concat(virtualSearchResults)} joinCardLists /> diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/use_virtual_search_results.ts b/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/use_virtual_search_results.ts new file mode 100644 index 00000000000000..5ee39768d8a583 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/onboarding_flow_form/use_virtual_search_results.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { CustomCard } from '../packages_list/types'; + +export function useVirtualSearchResults(): [CustomCard] { + const { + services: { application }, + } = useKibana(); + const getUrlForApp = application?.getUrlForApp; + + return [ + { + id: 'upload-file-virtual', + type: 'virtual', + title: i18n.translate('xpack.observability_onboarding.packageList.uploadFileTitle', { + defaultMessage: 'Upload a file', + }), + description: i18n.translate( + 'xpack.observability_onboarding.packageList.uploadFileDescription', + { + defaultMessage: + 'Upload data from a CSV, TSV, JSON or other log file to Elasticsearch for analysis.', + } + ), + name: 'upload-file', + categories: [], + icons: [ + { + type: 'eui', + src: 'addDataApp', + }, + ], + url: `${getUrlForApp?.('home')}#/tutorial_directory/fileDataViz`, + version: '', + integration: '', + isCollectionCard: false, + }, + ]; +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/packages_list/index.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/packages_list/index.tsx index 0e043d113423c9..4a204d0bc56013 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/packages_list/index.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/packages_list/index.tsx @@ -62,7 +62,7 @@ const PackageListGridWrapper = ({ }: WrapperProps) => { const customMargin = useCustomMargin(); const { filteredCards, isLoading } = useAvailablePackages({ - prereleaseIntegrationsEnabled: true, + prereleaseIntegrationsEnabled: false, }); const list: IntegrationCardItem[] = useIntegrationCardList( @@ -97,8 +97,10 @@ const PackageListGridWrapper = ({ box={{ incremental: true, }} - onChange={(arg) => { - setSearchQuery?.(arg.queryText); + onChange={({ queryText, error }) => { + if (error) return; + + setSearchQuery?.(queryText); }} query={searchQuery ?? ''} /> From f1ecb18d2e1582a5febdfe4644ebf5327a852c00 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Thu, 25 Apr 2024 06:35:03 -0700 Subject: [PATCH 101/138] [ML UI] Remove usage of deprecated React rendering utilities (#181094) ## Summary Partially addresses https://github.com/elastic/kibana-team/issues/805 These changes come up from searching in the code and finding where certain kinds of deprecated AppEx-SharedUX modules are imported. **Reviewers: Please interact with critical paths through the UI components touched in this PR, ESPECIALLY in terms of testing dark mode and i18n.** This focuses on code within **ML UI**. image Note: this also makes inclusion of `i18n` and `analytics` dependencies consistent. Analytics is an optional dependency for the SharedUX modules, which wrap `KibanaErrorBoundaryProvider` and is designed to capture telemetry about errors that are caught in the error boundary. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../change_point_detection_root.tsx | 6 +- .../log_categorization/show_flyout.tsx | 14 +- .../embeddable_change_point_chart.tsx | 45 +++-- .../embeddable_change_point_chart_factory.ts | 5 +- .../embeddable/handle_explicit_input.tsx | 2 +- x-pack/plugins/aiops/tsconfig.json | 3 +- .../data_drift/data_drift_app_state.tsx | 8 +- .../file_data_visualizer.tsx | 7 +- .../grid_embeddable/grid_embeddable.tsx | 36 ++-- .../index_data_visualizer.tsx | 53 +++--- x-pack/plugins/data_visualizer/tsconfig.json | 4 +- x-pack/plugins/ml/public/application/app.tsx | 43 +++-- .../header_menu_portal/header_menu_portal.tsx | 6 +- .../action_clone/clone_action_name.tsx | 5 +- .../jobs_list_page/jobs_list_page.tsx | 165 +++++++++--------- .../model_management/deployment_setup.tsx | 7 +- .../model_management/force_stop_dialog.tsx | 6 +- .../model_management/model_actions.tsx | 12 +- .../routing/routes/explorer/state_manager.tsx | 49 +++--- .../application/routing/use_active_route.tsx | 6 +- .../anomaly_charts_embeddable.tsx | 60 +++---- .../anomaly_charts_setup_flyout.tsx | 4 +- .../anomaly_swimlane_embeddable_factory.tsx | 109 ++++++------ .../anomaly_swimlane_setup_flyout.tsx | 4 +- .../common/resolve_job_selection.tsx | 5 +- .../job_creation/common/create_flyout.tsx | 5 +- .../single_metric_viewer_embeddable.tsx | 65 ++++--- .../single_metric_viewer_setup_flyout.tsx | 4 +- x-pack/plugins/ml/tsconfig.json | 2 +- x-pack/plugins/transform/public/app/app.tsx | 29 ++- .../components/toast_notification_text.tsx | 4 +- .../public/app/hooks/use_create_transform.tsx | 7 +- .../public/app/hooks/use_delete_transform.tsx | 28 +-- .../app/hooks/use_reauthorize_transform.tsx | 7 +- .../public/app/hooks/use_reset_transform.tsx | 15 +- .../app/hooks/use_schedule_now_transform.tsx | 7 +- .../public/app/hooks/use_start_transform.tsx | 7 +- .../public/app/hooks/use_stop_transform.tsx | 7 +- .../step_create/step_create_form.tsx | 12 +- .../step_details/step_details_form.tsx | 24 +-- .../edit_transform_retention_policy.tsx | 4 +- x-pack/plugins/transform/tsconfig.json | 3 +- 42 files changed, 422 insertions(+), 472 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_root.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_root.tsx index f8f23b3c19ef30..a259e3debfb040 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_root.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_root.tsx @@ -10,7 +10,7 @@ import React, { useMemo } from 'react'; import type { Observable } from 'rxjs'; import { map } from 'rxjs'; import { pick } from 'lodash'; -import { EuiThemeProvider as StyledComponentsThemeProvider } from '@kbn/kibana-react-plugin/common'; +import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; import { EuiSpacer } from '@elastic/eui'; import type { DataView } from '@kbn/data-views-plugin/common'; @@ -90,7 +90,7 @@ export const ChangePointDetectionAppState: FC return ( - + @@ -114,7 +114,7 @@ export const ChangePointDetectionAppState: FC - + ); }; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/show_flyout.tsx b/x-pack/plugins/aiops/public/components/log_categorization/show_flyout.tsx index 843f48400b00d3..87a2e9e934cc36 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/show_flyout.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/show_flyout.tsx @@ -34,8 +34,7 @@ export async function showCategorizeFlyout( originatingApp: string, additionalFilter?: CategorizationAdditionalFilter ): Promise { - const { analytics, http, theme, overlays, application, notifications, uiSettings, i18n } = - coreStart; + const { overlays, application, i18n } = coreStart; return new Promise(async (resolve, reject) => { try { @@ -45,15 +44,10 @@ export async function showCategorizeFlyout( }; const appDependencies: AiopsAppDependencies = { - analytics, - notifications, - uiSettings, - http, - theme, - application, - i18n, + ...coreStart, ...plugins, }; + const startServices = pick(coreStart, 'analytics', 'i18n', 'theme'); const datePickerDeps: DatePickerDependencies = { ...pick(appDependencies, ['data', 'http', 'notifications', 'theme', 'uiSettings']), i18n, @@ -82,7 +76,7 @@ export async function showCategorizeFlyout( , - { theme, i18n } + startServices ), { 'data-test-subj': 'aiopsCategorizeFlyout', diff --git a/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart.tsx b/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart.tsx index 101b9331e124cb..c76eae5c918d04 100644 --- a/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart.tsx +++ b/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart.tsx @@ -9,8 +9,7 @@ import React, { Suspense } from 'react'; import ReactDOM from 'react-dom'; import type { EmbeddableInput, EmbeddableOutput, IContainer } from '@kbn/embeddable-plugin/public'; import { Embeddable as AbstractEmbeddable } from '@kbn/embeddable-plugin/public'; -import { KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; -import type { ThemeServiceStart } from '@kbn/core-theme-browser'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { UI_SETTINGS } from '@kbn/data-plugin/public'; import type { IUiSettingsClient } from '@kbn/core/public'; @@ -34,7 +33,8 @@ export type EmbeddableChangePointChartInput = EmbeddableInput & EmbeddableChange export type EmbeddableChangePointChartOutput = EmbeddableOutput & { indexPatterns?: DataView[] }; export interface EmbeddableChangePointChartDeps { - theme: ThemeServiceStart; + analytics: CoreStart['analytics']; + theme: CoreStart['theme']; data: DataPublicPluginStart; uiSettings: IUiSettingsClient; http: CoreStart['http']; @@ -116,8 +116,7 @@ export class EmbeddableChangePointChart extends AbstractEmbeddable< // test subject selector for functional tests this.node.setAttribute('data-test-subj', 'aiopsEmbeddableChangePointChart'); - const I18nContext = this.deps.i18n.Context; - + const startServices = pick(this.deps, 'analytics', 'i18n', 'theme'); const datePickerDeps = { ...pick(this.deps, ['data', 'http', 'notifications', 'theme', 'uiSettings', 'i18n']), uiSettingsKeys: UI_SETTINGS, @@ -132,25 +131,23 @@ export class EmbeddableChangePointChart extends AbstractEmbeddable< } as unknown as AiopsAppDependencies; ReactDOM.render( - - - - - - - - - - - , + + + + + + + + + , el ); } diff --git a/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart_factory.ts b/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart_factory.ts index 48e3198cd80d17..318cc88fed6fd0 100644 --- a/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart_factory.ts +++ b/x-pack/plugins/aiops/public/embeddable/embeddable_change_point_chart_factory.ts @@ -68,22 +68,21 @@ export class EmbeddableChangePointChartFactory implements EmbeddableFactoryDefin async create(input: EmbeddableChangePointChartInput, parent?: IContainer) { try { const [ - { i18n: i18nService, theme, http, uiSettings, notifications }, + { http, uiSettings, notifications, ...startServices }, { lens, data, usageCollection, fieldFormats }, ] = await this.getStartServices(); return new EmbeddableChangePointChart( this.type, { - theme, http, - i18n: i18nService, uiSettings, data, notifications, lens, usageCollection, fieldFormats, + ...startServices, }, input, parent diff --git a/x-pack/plugins/aiops/public/embeddable/handle_explicit_input.tsx b/x-pack/plugins/aiops/public/embeddable/handle_explicit_input.tsx index cd64daee4b8492..35a8fb60b685e8 100644 --- a/x-pack/plugins/aiops/public/embeddable/handle_explicit_input.tsx +++ b/x-pack/plugins/aiops/public/embeddable/handle_explicit_input.tsx @@ -46,7 +46,7 @@ export async function resolveEmbeddableChangePointUserInput( }} /> , - { theme: coreStart.theme, i18n: coreStart.i18n } + coreStart ) ); } catch (error) { diff --git a/x-pack/plugins/aiops/tsconfig.json b/x-pack/plugins/aiops/tsconfig.json index a66b49cda17e29..6b11a3c6b91f42 100644 --- a/x-pack/plugins/aiops/tsconfig.json +++ b/x-pack/plugins/aiops/tsconfig.json @@ -54,7 +54,6 @@ "@kbn/utility-types", "@kbn/presentation-util-plugin", "@kbn/embeddable-plugin", - "@kbn/core-theme-browser", "@kbn/core-lifecycle-browser", "@kbn/cases-plugin", "@kbn/react-kibana-mount", @@ -69,6 +68,8 @@ "@kbn/aiops-log-rate-analysis", "@kbn/aiops-log-pattern-analysis", "@kbn/aiops-change-point-detection", + "@kbn/react-kibana-context-theme", + "@kbn/react-kibana-context-render", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_app_state.tsx b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_app_state.tsx index 154fee5b87606e..64b89251fd50a3 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_app_state.tsx +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_app_state.tsx @@ -16,7 +16,8 @@ import { UrlStateProvider } from '@kbn/ml-url-state'; import { Storage } from '@kbn/kibana-utils-plugin/public'; import { DatePickerContextProvider } from '@kbn/ml-date-picker'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { useLocation } from 'react-router-dom'; import { parse } from 'query-string'; @@ -91,6 +92,7 @@ export const DataDriftDetectionAppState: FC = ( charts, unifiedSearch, }; + const startServices = pick(coreStart, 'analytics', 'i18n', 'theme'); const datePickerDeps = { ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings', 'i18n']), uiSettingsKeys: UI_SETTINGS, @@ -128,7 +130,7 @@ export const DataDriftDetectionAppState: FC = ( }); return ( - + @@ -148,6 +150,6 @@ export const DataDriftDetectionAppState: FC = ( - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx index 03406ddb8045b4..3b97aa6474e74b 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx @@ -7,7 +7,8 @@ import '../_index.scss'; import type { FC } from 'react'; import React from 'react'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import type { ResultLinks } from '../../../common/app'; import { getCoreStart, getPluginsStart } from '../../kibana_services'; @@ -42,7 +43,7 @@ export const FileDataVisualizer: FC = ({ getAdditionalLinks, resultLinks const CloudContext = cloud?.CloudContextProvider || EmptyContext; return ( - + = ({ getAdditionalLinks, resultLinks /> - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index deeffbb934527b..1b0fcb2d688d21 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -18,7 +18,8 @@ import { FormattedMessage } from '@kbn/i18n-react'; import type { IContainer } from '@kbn/embeddable-plugin/public'; import { Embeddable } from '@kbn/embeddable-plugin/public'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { DatePickerContextProvider } from '@kbn/ml-date-picker'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import type { DataVisualizerStartDependencies } from '../../../../plugin'; @@ -107,31 +108,28 @@ export class DataVisualizerGridEmbeddable extends Embeddable< super.render(node); this.node = node; - const I18nContext = this.services[0].i18n.Context; - const services = { ...this.services[0], ...this.services[1] }; + const startServices = pick(this.services[0], 'analytics', 'i18n', 'theme'); const datePickerDeps = { ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings', 'i18n']), uiSettingsKeys: UI_SETTINGS, }; ReactDOM.render( - - - - - }> - this.updateOutput(output)} - /> - - - - - , + + + + }> + this.updateOutput(output)} + /> + + + + , node ); } diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index 6a8e043bae37a9..7482e686db33ef 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -14,7 +14,9 @@ import { isEqual } from 'lodash'; import { encode } from '@kbn/rison'; import { i18n } from '@kbn/i18n'; import { Storage } from '@kbn/kibana-utils-plugin/public'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; +import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; import { StorageContextProvider } from '@kbn/ml-local-storage'; import type { DataView } from '@kbn/data-views-plugin/public'; import { getNestedProperty } from '@kbn/ml-nested-property'; @@ -334,6 +336,7 @@ export const IndexDataVisualizer: FC = ({ unifiedSearch, }; + const startServices = pick(coreStart, 'analytics', 'i18n', 'theme'); const datePickerDeps: DatePickerDependencies = { ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings', 'i18n']), uiSettingsKeys: UI_SETTINGS, @@ -341,28 +344,30 @@ export const IndexDataVisualizer: FC = ({ }; return ( - - - - - {!esql ? ( - - ) : ( - - )} - - - - + + + + + + {!esql ? ( + + ) : ( + + )} + + + + + ); }; diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json index a6a865c86818c4..428311f87b305b 100644 --- a/x-pack/plugins/data_visualizer/tsconfig.json +++ b/x-pack/plugins/data_visualizer/tsconfig.json @@ -78,7 +78,9 @@ "@kbn/visualization-utils", "@kbn/ml-time-buckets", "@kbn/aiops-log-rate-analysis", - "@kbn/discover-utils" + "@kbn/discover-utils", + "@kbn/react-kibana-context-render", + "@kbn/react-kibana-context-theme" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index 05e18a66a72fec..7a965a1541855c 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -14,7 +14,8 @@ import type { AppMountParameters, CoreStart } from '@kbn/core/public'; import { DatePickerContextProvider, type DatePickerDependencies } from '@kbn/ml-date-picker'; import { Storage } from '@kbn/kibana-utils-plugin/public'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { StorageContextProvider } from '@kbn/ml-local-storage'; import useLifecycles from 'react-use/lib/useLifecycles'; import useObservable from 'react-use/lib/useObservable'; @@ -118,37 +119,35 @@ const App: FC = ({ if (!licenseReady || !mlCapabilities) return null; + const startServices = pick(coreStart, 'analytics', 'i18n', 'theme'); const datePickerDeps: DatePickerDependencies = { ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings', 'i18n']), uiSettingsKeys: UI_SETTINGS, showFrozenDataTierChoice: !isServerless, }; - const I18nContext = coreStart.i18n.Context; const ApplicationUsageTrackingProvider = deps.usageCollection?.components.ApplicationUsageTrackingProvider ?? React.Fragment; return ( - - - - - - - - - - - - - - - + + + + + + + + + + + + + ); }; diff --git a/x-pack/plugins/ml/public/application/components/header_menu_portal/header_menu_portal.tsx b/x-pack/plugins/ml/public/application/components/header_menu_portal/header_menu_portal.tsx index 5903b0a81eac78..f80c8888ee1814 100644 --- a/x-pack/plugins/ml/public/application/components/header_menu_portal/header_menu_portal.tsx +++ b/x-pack/plugins/ml/public/application/components/header_menu_portal/header_menu_portal.tsx @@ -19,7 +19,6 @@ export interface HeaderMenuPortalProps { export const HeaderMenuPortal: FC = ({ children }) => { const { services } = useMlKibana(); - const { theme, i18n } = services; const { setHeaderActionMenu } = useContext(MlPageControlsContext); @@ -35,7 +34,7 @@ export const HeaderMenuPortal: FC = ({ children }) => { , - { theme, i18n } + services ); return mount(element); }); @@ -44,8 +43,7 @@ export const HeaderMenuPortal: FC = ({ children }) => { portalNode.unmount(); setHeaderActionMenu(undefined); }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [portalNode, setHeaderActionMenu, services.theme.theme$]); + }, [portalNode, setHeaderActionMenu, services]); return {children}; }; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx index 54ca887fcdce33..55f25f70253a74 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx @@ -414,8 +414,7 @@ export const useNavigateToWizardWithClonedJob = () => { data: { dataViews }, http: { basePath }, application: { capabilities }, - theme, - i18n: i18nStart, + ...startServices }, } = useMlKibana(); const navigateToPath = useNavigateToPath(); @@ -464,7 +463,7 @@ export const useNavigateToWizardWithClonedJob = () => { ) : null} , - { theme, i18n: i18nStart } + startServices ), }); } diff --git a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx index 4e12c16356401b..7f5929354b61f9 100644 --- a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx +++ b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx @@ -23,7 +23,8 @@ import { import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { ManagementAppMountParams } from '@kbn/management-plugin/public'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app'; import type { SharePluginStart } from '@kbn/share-plugin/public'; import type { SpacesContextProps, SpacesPluginStart } from '@kbn/spaces-plugin/public'; @@ -73,8 +74,6 @@ export const JobsListPage: FC = ({ const [isPlatinumOrTrialLicense, setIsPlatinumOrTrialLicense] = useState(true); const [showSyncFlyout, setShowSyncFlyout] = useState(false); const [currentTabId, setCurrentTabId] = useState('anomaly-detector'); - const I18nContext = coreStart.i18n.Context; - const theme$ = coreStart.theme.theme$; const mlServices = useMemo( () => getMlGlobalServices(coreStart.http, data.dataViews, usageCollection), @@ -122,87 +121,85 @@ export const JobsListPage: FC = ({ } return ( - - - - - - - - + + + + + + + } + description={ + + } + rightSideItems={[]} + bottomBorder + paddingSize={'none'} + /> + + + + + + + <> + setShowSyncFlyout(true)} + data-test-subj="mlStackMgmtSyncButton" + > + {i18n.translate('xpack.ml.management.jobsList.syncFlyoutButton', { + defaultMessage: 'Synchronize saved objects', + })} + + {showSyncFlyout && } + + + + + - } - description={ - - } - rightSideItems={[]} - bottomBorder - paddingSize={'none'} - /> - - - - - - - <> - setShowSyncFlyout(true)} - data-test-subj="mlStackMgmtSyncButton" - > - {i18n.translate('xpack.ml.management.jobsList.syncFlyoutButton', { - defaultMessage: 'Synchronize saved objects', - })} - - {showSyncFlyout && } - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + ); }; diff --git a/x-pack/plugins/ml/public/application/model_management/deployment_setup.tsx b/x-pack/plugins/ml/public/application/model_management/deployment_setup.tsx index 368cb73e66071a..cef8c87cdea293 100644 --- a/x-pack/plugins/ml/public/application/model_management/deployment_setup.tsx +++ b/x-pack/plugins/ml/public/application/model_management/deployment_setup.tsx @@ -28,7 +28,7 @@ import { EuiSelect, EuiSpacer, } from '@elastic/eui'; -import type { I18nStart, OverlayStart, ThemeServiceStart } from '@kbn/core/public'; +import type { CoreStart, OverlayStart } from '@kbn/core/public'; import { css } from '@emotion/react'; import { numberValidator } from '@kbn/ml-agg-utils'; import { toMountPoint } from '@kbn/react-kibana-mount'; @@ -529,8 +529,7 @@ export const StartUpdateDeploymentModal: FC = ({ export const getUserInputModelDeploymentParamsProvider = ( overlays: OverlayStart, - theme: ThemeServiceStart, - i18nStart: I18nStart, + startServices: Pick, startModelDeploymentDocUrl: string ) => ( @@ -563,7 +562,7 @@ export const getUserInputModelDeploymentParamsProvider = resolve(); }} />, - { theme, i18n: i18nStart } + startServices ) ); } catch (e) { diff --git a/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx b/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx index 9f50c2881d6d64..171143a581194d 100644 --- a/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx +++ b/x-pack/plugins/ml/public/application/model_management/force_stop_dialog.tsx @@ -10,7 +10,7 @@ import type { EuiCheckboxGroupOption } from '@elastic/eui'; import { EuiCallOut, EuiCheckboxGroup, EuiConfirmModal, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; -import type { I18nStart, OverlayStart, ThemeServiceStart } from '@kbn/core/public'; +import type { CoreStart, OverlayStart } from '@kbn/core/public'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { isDefined } from '@kbn/ml-is-defined'; import { toMountPoint } from '@kbn/react-kibana-mount'; @@ -219,7 +219,7 @@ export const StopModelDeploymentsConfirmDialog: FC + (overlays: OverlayStart, startServices: Pick) => async (forceStopModel: ModelItem): Promise => { return new Promise(async (resolve, reject) => { try { @@ -236,7 +236,7 @@ export const getUserConfirmationProvider = resolve(deploymentIds); }} />, - { theme, i18n: i18nStart } + startServices ) ); } catch (e) { diff --git a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx index 5d690a7a58fd89..b9f17084bd81ff 100644 --- a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx +++ b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx @@ -58,10 +58,9 @@ export function useModelActions({ services: { application: { navigateToUrl }, overlays, - theme, - i18n: i18nStart, docLinks, mlServices: { mlApiServices }, + ...startServices }, } = useMlKibana(); @@ -109,19 +108,18 @@ export function useModelActions({ }, [mlApiServices]); const getUserConfirmation = useMemo( - () => getUserConfirmationProvider(overlays, theme, i18nStart), - [i18nStart, overlays, theme] + () => getUserConfirmationProvider(overlays, startServices), + [overlays, startServices] ); const getUserInputModelDeploymentParams = useMemo( () => getUserInputModelDeploymentParamsProvider( overlays, - theme, - i18nStart, + startServices, startModelDeploymentDocUrl ), - [overlays, theme, i18nStart, startModelDeploymentDocUrl] + [overlays, startServices, startModelDeploymentDocUrl] ); const isBuiltInModel = useCallback( diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer/state_manager.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer/state_manager.tsx index bcb5c200cdf074..1c32c298047113 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer/state_manager.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer/state_manager.tsx @@ -12,7 +12,6 @@ import useObservable from 'react-use/lib/useObservable'; import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { EuiThemeProvider as StyledComponentsThemeProvider } from '@kbn/kibana-react-plugin/common'; import { useUrlState } from '@kbn/ml-url-state'; import { useTimefilter } from '@kbn/ml-date-picker'; import { ML_JOB_ID } from '@kbn/ml-anomaly-utils'; @@ -217,31 +216,29 @@ export const ExplorerUrlStateManager: FC = ({ - - - - {jobsWithTimeRange.length === 0 ? ( - - ) : ( - - )} - - - + + + {jobsWithTimeRange.length === 0 ? ( + + ) : ( + + )} + +
); }; diff --git a/x-pack/plugins/ml/public/application/routing/use_active_route.tsx b/x-pack/plugins/ml/public/application/routing/use_active_route.tsx index 3974b13eb51de7..202c552559c1bb 100644 --- a/x-pack/plugins/ml/public/application/routing/use_active_route.tsx +++ b/x-pack/plugins/ml/public/application/routing/use_active_route.tsx @@ -24,7 +24,7 @@ export const useActiveRoute = (routesList: MlRoute[]): MlRoute => { const { pathname } = useLocation(); const { - services: { executionContext, overlays, theme, i18n }, + services: { executionContext, overlays, ...startServices }, } = useMlKibana(); /** @@ -78,7 +78,7 @@ export const useActiveRoute = (routesList: MlRoute[]): MlRoute => { />

, - { theme, i18n } + startServices ) ); @@ -90,7 +90,7 @@ export const useActiveRoute = (routesList: MlRoute[]): MlRoute => { }, 15000); } }, - [activeRoute, overlays, theme, pathname, i18n] + [activeRoute, overlays, pathname, startServices] ); useExecutionContext(executionContext, { diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx index ded0f536ae96ae..9d7b08fa935d09 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable.tsx @@ -10,7 +10,8 @@ import ReactDOM from 'react-dom'; import type { CoreStart } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import { Subject, Subscription, type BehaviorSubject } from 'rxjs'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import type { IContainer } from '@kbn/embeddable-plugin/public'; import { embeddableInputToSubject } from '@kbn/embeddable-plugin/public'; import { embeddableOutputToSubject } from '@kbn/embeddable-plugin/public'; @@ -91,38 +92,33 @@ export class AnomalyChartsEmbeddable extends AnomalyDetectionEmbeddable< // required for the export feature to work this.node.setAttribute('data-shared-item', ''); - const I18nContext = this.services[0].i18n.Context; - const theme$ = this.services[0].theme.theme$; - ReactDOM.render( - - - - }> - - - - - , + + + }> + + + + , node ); } diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_setup_flyout.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_setup_flyout.tsx index 2dd09620dbbd18..0244f949e467af 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_setup_flyout.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_setup_flyout.tsx @@ -23,7 +23,7 @@ export async function resolveEmbeddableAnomalyChartsUserInput( dataViews: DataViewsContract, input?: AnomalyChartsEmbeddableInput ): Promise> { - const { http, overlays, theme, i18n } = coreStart; + const { http, overlays, ...startServices } = coreStart; const { getJobs } = mlApiServicesProvider(new HttpService(http)); @@ -52,7 +52,7 @@ export async function resolveEmbeddableAnomalyChartsUserInput( reject(); }} />, - { theme, i18n } + startServices ) ); } catch (error) { diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.tsx index 665e0b99ddd72b..c333869dd82955 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.tsx @@ -21,7 +21,7 @@ import { initializeTitles, useBatchedPublishingSubjects, } from '@kbn/presentation-publishing'; -import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import React, { useCallback, useState } from 'react'; import useUnmount from 'react-use/lib/useUnmount'; import type { Observable } from 'rxjs'; @@ -210,10 +210,9 @@ export const getAnomalySwimLaneEmbeddableFactory = ( return { api, Component: () => { - const { theme, i18n, uiSettings } = coreStartServices; + const { uiSettings } = coreStartServices; const { uiActions } = pluginsStartServices; - const I18nContext = i18n.Context; const timeBuckets = useTimeBuckets(uiSettings); if (!apiHasExecutionContext(parentApi)) { @@ -279,61 +278,59 @@ export const getAnomalySwimLaneEmbeddableFactory = ( } return ( - - - -
- + +
+ chartWidth$.next(size)} + selection={selectedCells} + onCellsSelection={onCellsSelection} + onPaginationChange={(update) => { + if (update.fromPage) { + api.updatePagination({ fromPage: update.fromPage }); } - onResize={(size) => chartWidth$.next(size)} - selection={selectedCells} - onCellsSelection={onCellsSelection} - onPaginationChange={(update) => { - if (update.fromPage) { - api.updatePagination({ fromPage: update.fromPage }); - } - if (update.perPage) { - api.updatePagination({ perPage: update.perPage, fromPage: 1 }); - } - }} - isLoading={dataLoading.value!} - yAxisWidth={{ max: Y_AXIS_LABEL_WIDTH }} - noDataWarning={ - - - - } - /> + if (update.perPage) { + api.updatePagination({ perPage: update.perPage, fromPage: 1 }); } - chartsService={pluginsStartServices.charts} - onRenderComplete={onRenderComplete} - /> -
- - - + }} + isLoading={dataLoading.value!} + yAxisWidth={{ max: Y_AXIS_LABEL_WIDTH }} + noDataWarning={ + + + + } + /> + } + chartsService={pluginsStartServices.charts} + onRenderComplete={onRenderComplete} + /> +
+ + ); }, }; diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx index ed795ab0f9ce52..596da35f66d4cc 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx @@ -23,7 +23,7 @@ export async function resolveAnomalySwimlaneUserInput( dataViews: DataViewsContract, input?: Partial ): Promise { - const { http, overlays, theme, i18n } = coreStart; + const { http, overlays, ...startServices } = coreStart; const { getJobs } = mlApiServicesProvider(new HttpService(http)); @@ -52,7 +52,7 @@ export async function resolveAnomalySwimlaneUserInput( reject(); }} />, - { theme, i18n } + startServices ) ); } catch (error) { diff --git a/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx b/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx index 9d6c394a8e4e2d..99b301ee19e9f1 100644 --- a/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx +++ b/x-pack/plugins/ml/public/embeddables/common/resolve_job_selection.tsx @@ -34,9 +34,8 @@ export async function resolveJobSelection( const { http, uiSettings, - theme, - i18n, application: { currentAppId$ }, + ...startServices } = coreStart; return new Promise(async (resolve, reject) => { @@ -86,7 +85,7 @@ export async function resolveJobSelection( maps={maps} /> , - { theme, i18n } + startServices ), { 'data-test-subj': 'mlFlyoutJobSelector', diff --git a/x-pack/plugins/ml/public/embeddables/job_creation/common/create_flyout.tsx b/x-pack/plugins/ml/public/embeddables/job_creation/common/create_flyout.tsx index 461f67a47aa2ee..5bab4967f0cdb1 100644 --- a/x-pack/plugins/ml/public/embeddables/job_creation/common/create_flyout.tsx +++ b/x-pack/plugins/ml/public/embeddables/job_creation/common/create_flyout.tsx @@ -31,10 +31,9 @@ export function createFlyout( ): Promise { const { http, - theme, - i18n, overlays, application: { currentAppId$ }, + ...startServices } = coreStart; return new Promise(async (resolve, reject) => { @@ -63,7 +62,7 @@ export function createFlyout( }} /> , - { theme, i18n } + startServices ), { 'data-test-subj': 'mlFlyoutLayerSelector', diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx index 8cbd2ad30ae8e1..c0d7902c5ab66b 100644 --- a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_embeddable.tsx @@ -14,7 +14,8 @@ import { Subject, Subscription, type BehaviorSubject } from 'rxjs'; import type { CoreStart } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import type { IContainer } from '@kbn/embeddable-plugin/public'; import { DatePickerContextProvider, type DatePickerDependencies } from '@kbn/ml-date-picker'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; @@ -113,9 +114,7 @@ export class SingleMetricViewerEmbeddable extends Embeddable< // required for the export feature to work this.node.setAttribute('data-shared-item', ''); - const I18nContext = this.services[0].i18n.Context; - const theme$ = this.services[0].theme.theme$; - + const startServices = pick(this.services[0], 'analytics', 'i18n', 'theme'); const datePickerDeps: DatePickerDependencies = { ...pick(this.services[0], ['http', 'notifications', 'theme', 'uiSettings', 'i18n']), data: this.services[1].data, @@ -124,36 +123,34 @@ export class SingleMetricViewerEmbeddable extends Embeddable< }; ReactDOM.render( - - - - - }> - - - - - - , + + + + }> + + + + + , node ); } diff --git a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx index 11927e9bca377f..92e77f788fc3f4 100644 --- a/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx +++ b/x-pack/plugins/ml/public/embeddables/single_metric_viewer/single_metric_viewer_setup_flyout.tsx @@ -22,7 +22,7 @@ export async function resolveEmbeddableSingleMetricViewerUserInput( mlApiServices: MlApiServices, input?: SingleMetricViewerEmbeddableInput ): Promise> { - const { overlays, theme, i18n } = coreStart; + const { overlays, ...startServices } = coreStart; const timefilter = pluginStart.data.query.timefilter.timefilter; return new Promise(async (resolve, reject) => { @@ -62,7 +62,7 @@ export async function resolveEmbeddableSingleMetricViewerUserInput( }} /> , - { theme, i18n } + startServices ) ); } catch (error) { diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index 16ab6806868a57..dba33a6043bc81 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -121,10 +121,10 @@ "@kbn/ml-time-buckets", "@kbn/aiops-change-point-detection", "@kbn/inference_integration_flyout", - "@kbn/react-kibana-context-theme", "@kbn/presentation-containers", "@kbn/presentation-panel-plugin", "@kbn/shared-ux-utility", "@kbn/discover-utils", + "@kbn/react-kibana-context-render", ], } diff --git a/x-pack/plugins/transform/public/app/app.tsx b/x-pack/plugins/transform/public/app/app.tsx index 9da8af2ba84074..c6481f74404259 100644 --- a/x-pack/plugins/transform/public/app/app.tsx +++ b/x-pack/plugins/transform/public/app/app.tsx @@ -9,11 +9,10 @@ import React, { type FC } from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; -import { EuiErrorBoundary } from '@elastic/eui'; - import { Router, Routes, Route } from '@kbn/shared-ux-router'; import type { ScopedHistory } from '@kbn/core/public'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import type { ExperimentalFeatures } from '../../common/config'; import { SECTION_SLUG } from './common/constants'; @@ -49,8 +48,6 @@ export const renderApp = ( enabledFeatures: TransformEnabledFeatures, experimentalFeatures: ExperimentalFeatures ) => { - const I18nContext = appDependencies.i18n.Context; - const queryClient = new QueryClient({ defaultOptions: { queries: { @@ -61,21 +58,17 @@ export const renderApp = ( }); render( - + - - - - - - - - - - - + + + + + + + - , + , element ); diff --git a/x-pack/plugins/transform/public/app/components/toast_notification_text.tsx b/x-pack/plugins/transform/public/app/components/toast_notification_text.tsx index 8b2f154e8124e9..aecca492671779 100644 --- a/x-pack/plugins/transform/public/app/components/toast_notification_text.tsx +++ b/x-pack/plugins/transform/public/app/components/toast_notification_text.tsx @@ -38,7 +38,7 @@ export const ToastNotificationText: FC = ({ inline = false, forceModal = false, }) => { - const { overlays, theme, i18n: i18nStart } = useAppDependencies(); + const { overlays, ...startServices } = useAppDependencies(); if (!forceModal && typeof text === 'string' && text.length <= MAX_SIMPLE_MESSAGE_LENGTH) { return text; @@ -87,7 +87,7 @@ export const ToastNotificationText: FC = ({ , - { theme, i18n: i18nStart } + startServices ) ); }; diff --git a/x-pack/plugins/transform/public/app/hooks/use_create_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_create_transform.tsx index c678549683becb..53682db36c0291 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_create_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_create_transform.tsx @@ -32,7 +32,7 @@ interface CreateTransformArgs { } export const useCreateTransform = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -42,10 +42,7 @@ export const useCreateTransform = () => { defaultMessage: 'An error occurred creating the transform {transformId}:', values: { transformId }, }), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint(, startServices), }); } diff --git a/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx index 9f2cf91b088b1d..c2549b0edad56d 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx @@ -87,7 +87,7 @@ export const useDeleteIndexAndTargetIndex = (items: TransformListRow[]) => { }; export const useDeleteTransforms = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -104,7 +104,7 @@ export const useDeleteTransforms = () => { }), text: toMountPoint( , - { theme, i18n: i18nStart } + startServices ), }), onSuccess: (results) => { @@ -121,10 +121,10 @@ export const useDeleteTransforms = () => { defaultMessage: 'An error occurred deleting the transform {transformId}', values: { transformId }, }), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } @@ -138,10 +138,10 @@ export const useDeleteTransforms = () => { values: { destinationIndex }, } ), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } @@ -155,10 +155,10 @@ export const useDeleteTransforms = () => { values: { destinationIndex }, } ), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } } diff --git a/x-pack/plugins/transform/public/app/hooks/use_reauthorize_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_reauthorize_transform.tsx index 9ecd1b8717243b..0465a178c09a3b 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_reauthorize_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_reauthorize_transform.tsx @@ -25,7 +25,7 @@ import { ToastNotificationText } from '../components'; import { useRefreshTransformList } from './use_refresh_transform_list'; export const useReauthorizeTransforms = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -46,10 +46,7 @@ export const useReauthorizeTransforms = () => { defaultMessage: 'An error occurred calling the reauthorize transforms request.', } ), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint(, startServices), }), onSuccess: (results) => { for (const transformId in results) { diff --git a/x-pack/plugins/transform/public/app/hooks/use_reset_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_reset_transform.tsx index 1f415eae1ad203..fafc26581fa09e 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_reset_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_reset_transform.tsx @@ -24,7 +24,7 @@ import { ToastNotificationText } from '../components'; import { useRefreshTransformList } from './use_refresh_transform_list'; export const useResetTransforms = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -41,10 +41,7 @@ export const useResetTransforms = () => { }), text: toMountPoint( , - { - theme, - i18n: i18nStart, - } + startServices ), }), onSuccess: (results) => { @@ -60,10 +57,10 @@ export const useResetTransforms = () => { defaultMessage: 'An error occurred resetting the transform {transformId}', values: { transformId }, }), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } } diff --git a/x-pack/plugins/transform/public/app/hooks/use_schedule_now_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_schedule_now_transform.tsx index bb568673a57587..22201b28068483 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_schedule_now_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_schedule_now_transform.tsx @@ -24,7 +24,7 @@ import { ToastNotificationText } from '../components'; import { useRefreshTransformList } from './use_refresh_transform_list'; export const useScheduleNowTransforms = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -46,10 +46,7 @@ export const useScheduleNowTransforms = () => { 'An error occurred calling the request to schedule the transform to process data instantly.', } ), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint(, startServices), }), onSuccess: (results) => { for (const transformId in results) { diff --git a/x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx index 104c3145fc2591..2175ccffeea536 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx @@ -24,7 +24,7 @@ import { ToastNotificationText } from '../components'; import { useRefreshTransformList } from './use_refresh_transform_list'; export const useStartTransforms = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -42,10 +42,7 @@ export const useStartTransforms = () => { defaultMessage: 'An error occurred calling the start transforms request.', } ), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint(, startServices), }), onSuccess: (results) => { for (const transformId in results) { diff --git a/x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx b/x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx index 564b17feac3f98..439c654ded1691 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_stop_transform.tsx @@ -25,7 +25,7 @@ import { ToastNotificationText } from '../components'; import { useRefreshTransformList } from './use_refresh_transform_list'; export const useStopTransforms = () => { - const { http, i18n: i18nStart, theme } = useAppDependencies(); + const { http, ...startServices } = useAppDependencies(); const refreshTransformList = useRefreshTransformList(); const toastNotifications = useToastNotifications(); @@ -43,10 +43,7 @@ export const useStopTransforms = () => { defaultMessage: 'An error occurred called the stop transforms request.', } ), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint(, startServices), }), onSuccess: (results) => { for (const transformId in results) { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx index 35e22d07fe793a..e15dec45908a4b 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx @@ -84,7 +84,7 @@ export const StepCreateForm: FC = React.memo( const [discoverLink, setDiscoverLink] = useState(); const toastNotifications = useToastNotifications(); - const { application, i18n: i18nStart, share, theme } = useAppDependencies(); + const { application, share, ...startServices } = useAppDependencies(); const isDiscoverAvailable = application.capabilities.discover?.show ?? false; useEffect(() => { @@ -200,13 +200,13 @@ export const StepCreateForm: FC = React.memo( title: i18n.translate('xpack.transform.stepCreateForm.progressErrorMessage', { defaultMessage: 'An error occurred getting the progress percentage:', }), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } - }, [i18nStart, stats, theme, toastNotifications, transformConfig, transformId]); + }, [stats, toastNotifications, transformConfig, transformId, startServices]); function getTransformConfigDevConsoleStatement() { return `PUT _transform/${transformId}\n${JSON.stringify(transformConfig, null, 2)}\n\n`; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx index 9f60fe827ae600..416feac4c3c336 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx @@ -76,7 +76,7 @@ interface StepDetailsFormProps { export const StepDetailsForm: FC = React.memo( ({ overrides = {}, onChange, searchItems, stepDefineState }) => { - const { application, i18n: i18nStart, theme } = useAppDependencies(); + const { application, ...startServices } = useAppDependencies(); const { capabilities } = application; const toastNotifications = useToastNotifications(); const { esIndicesCreateIndex } = useDocumentationLinks(); @@ -167,10 +167,10 @@ export const StepDetailsForm: FC = React.memo( title: i18n.translate('xpack.transform.stepDetailsForm.errorGettingTransformList', { defaultMessage: 'An error occurred getting the existing transform IDs:', }), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } // custom comparison @@ -185,7 +185,7 @@ export const StepDetailsForm: FC = React.memo( }), text: toMountPoint( , - { theme, i18n: i18nStart } + startServices ), }); } @@ -202,10 +202,10 @@ export const StepDetailsForm: FC = React.memo( title: i18n.translate('xpack.transform.stepDetailsForm.errorGettingIndexNames', { defaultMessage: 'An error occurred getting the existing index names:', }), - text: toMountPoint(, { - theme, - i18n: i18nStart, - }), + text: toMountPoint( + , + startServices + ), }); } // custom comparison @@ -224,7 +224,7 @@ export const StepDetailsForm: FC = React.memo( }), text: toMountPoint( , - { theme, i18n: i18nStart } + startServices ), }); } @@ -242,7 +242,7 @@ export const StepDetailsForm: FC = React.memo( }), text: toMountPoint( , - { theme, i18n: i18nStart } + startServices ), }); } diff --git a/x-pack/plugins/transform/public/app/sections/edit_transform/components/edit_transform_retention_policy.tsx b/x-pack/plugins/transform/public/app/sections/edit_transform/components/edit_transform_retention_policy.tsx index c32409cb6ff7f9..b49eb67b0db08c 100644 --- a/x-pack/plugins/transform/public/app/sections/edit_transform/components/edit_transform_retention_policy.tsx +++ b/x-pack/plugins/transform/public/app/sections/edit_transform/components/edit_transform_retention_policy.tsx @@ -30,7 +30,7 @@ import { useRetentionPolicyField } from '../state_management/selectors/retention import { EditTransformFlyoutFormTextInput } from './edit_transform_flyout_form_text_input'; export const EditTransformRetentionPolicy: FC = () => { - const { i18n: i18nStart, theme } = useAppDependencies(); + const startServices = useAppDependencies(); const toastNotifications = useToastNotifications(); @@ -70,7 +70,7 @@ export const EditTransformRetentionPolicy: FC = () => { }), text: toMountPoint( , - { theme, i18n: i18nStart } + startServices ), }); } diff --git a/x-pack/plugins/transform/tsconfig.json b/x-pack/plugins/transform/tsconfig.json index faa4e2e67dd53b..91581e7dfc4ffd 100644 --- a/x-pack/plugins/transform/tsconfig.json +++ b/x-pack/plugins/transform/tsconfig.json @@ -74,7 +74,8 @@ "@kbn/ml-creation-wizard-utils", "@kbn/alerts-as-data-utils", "@kbn/code-editor", - "@kbn/rule-data-utils" + "@kbn/rule-data-utils", + "@kbn/react-kibana-context-render" ], "exclude": [ "target/**/*", From 4ca36e869e394a2aaf97c22f4c76f4b8a260c7c5 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Thu, 25 Apr 2024 06:35:34 -0700 Subject: [PATCH 102/138] [Enterprise Search UI] Remove usage of deprecated React rendering utilities (#181097) ## Summary Partially addresses https://github.com/elastic/kibana-team/issues/805 These changes come up from searching in the code and finding where certain kinds of deprecated AppEx-SharedUX modules are imported. **Reviewers: Please interact with critical paths through the UI components touched in this PR, ESPECIALLY in terms of testing dark mode and i18n.** This focuses on code within **Enterprise Search**. image Note: this also makes inclusion of `i18n` and `analytics` dependencies consistent. Analytics is an optional dependency for the SharedUX modules, which wrap `KibanaErrorBoundaryProvider` and is designed to capture telemetry about errors that are caught in the error boundary. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../shared/log_stream/log_stream.test.tsx | 53 ++++++++----------- .../shared/log_stream/log_stream.tsx | 7 +-- .../plugins/enterprise_search/tsconfig.json | 4 +- .../search_playground/public/application.tsx | 6 +-- .../plugins/search_playground/tsconfig.json | 4 +- .../public/application/connectors.tsx | 6 +-- .../public/application/elasticsearch.tsx | 23 ++++---- .../public/test/test_utils.tsx | 5 +- .../plugins/serverless_search/tsconfig.json | 1 + 9 files changed, 48 insertions(+), 61 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.test.tsx index be855aa1793afd..3d94a6e3b22e26 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.test.tsx @@ -9,6 +9,8 @@ import React from 'react'; import { shallow } from 'enzyme'; +import { LogStream } from '@kbn/logs-shared-plugin/public'; + import { EntSearchLogStream } from '.'; const fakeSourceId = 'fake-source-id'; @@ -17,15 +19,12 @@ describe('EntSearchLogStream', () => { const mockDateNow = jest.spyOn(global.Date, 'now').mockReturnValue(160000000); describe('renders with default props', () => { - /** As a result of the theme provider being added, we have to extract the child component to correctly assert */ const wrapper = shallow( - shallow( - - ).prop('children') + ); it('renders a LogStream (wrapped in React.Suspense) component', () => { - expect(wrapper.type()).toEqual(React.Suspense); + expect(wrapper.type()).toEqual(LogStream); }); it('renders with the empty sourceId', () => { @@ -41,13 +40,11 @@ describe('EntSearchLogStream', () => { describe('renders custom props', () => { it('overrides the default props', () => { const wrapper = shallow( - shallow( - - ).prop('children') + ); expect(wrapper.prop('logView')).toEqual({ type: 'log-view-reference', logViewId: 'test' }); @@ -57,12 +54,10 @@ describe('EntSearchLogStream', () => { it('allows passing a custom hoursAgo that modifies the default start timestamp', () => { const wrapper = shallow( - shallow( - - ).prop('children') + ); expect(wrapper.prop('startTimestamp')).toEqual(156400000); @@ -71,18 +66,16 @@ describe('EntSearchLogStream', () => { it('allows passing any prop that the LogStream component takes', () => { const wrapper = shallow( - shallow( - - ).prop('children') + ); expect(wrapper.prop('height')).toEqual(500); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.tsx index 79884fd02848f3..5748d306b6319f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/log_stream/log_stream.tsx @@ -7,7 +7,6 @@ import React from 'react'; -import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; import { LogStream, LogStreamProps } from '@kbn/logs-shared-plugin/public'; /* @@ -35,9 +34,5 @@ export const EntSearchLogStream: React.FC = ({ if (!endTimestamp) endTimestamp = Date.now(); if (!startTimestamp) startTimestamp = endTimestamp - hoursAgo * 60 * 60 * 1000; - return ( - - - - ); + return ; }; diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index 2750299a8f6a09..4aba94dff1bbcd 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -64,7 +64,6 @@ "@kbn/core-http-browser-mocks", "@kbn/core-application-browser", "@kbn/core-capabilities-common", - "@kbn/react-kibana-context-theme", "@kbn/code-editor", "@kbn/console-plugin", "@kbn/core-notifications-browser", @@ -73,6 +72,7 @@ "@kbn/search-playground", "@kbn/utility-types", "@kbn/index-management", - "@kbn/deeplinks-search" + "@kbn/deeplinks-search", + "@kbn/react-kibana-context-theme" ] } diff --git a/x-pack/plugins/search_playground/public/application.tsx b/x-pack/plugins/search_playground/public/application.tsx index f76a30a22ae87f..ef132487a493bd 100644 --- a/x-pack/plugins/search_playground/public/application.tsx +++ b/x-pack/plugins/search_playground/public/application.tsx @@ -8,7 +8,7 @@ import React from 'react'; import ReactDOM from 'react-dom'; import { CoreStart, AppMountParameters } from '@kbn/core/public'; -import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { I18nProvider } from '@kbn/i18n-react'; import { BrowserRouter as Router } from '@kbn/shared-ux-router'; @@ -27,7 +27,7 @@ export const renderApp = ( const navigation = services.navigation; ReactDOM.render( - + @@ -57,7 +57,7 @@ export const renderApp = ( - , + , element ); diff --git a/x-pack/plugins/search_playground/tsconfig.json b/x-pack/plugins/search_playground/tsconfig.json index 8bcef27ddb8f86..c4cfb17e288707 100644 --- a/x-pack/plugins/search_playground/tsconfig.json +++ b/x-pack/plugins/search_playground/tsconfig.json @@ -20,7 +20,6 @@ "@kbn/ml-response-stream", "@kbn/security-plugin", "@kbn/user-profile-components", - "@kbn/react-kibana-context-theme", "@kbn/shared-ux-router", "@kbn/shared-ux-page-kibana-template", "@kbn/navigation-plugin", @@ -34,7 +33,8 @@ "@kbn/cases-plugin", "@kbn/triggers-actions-ui-plugin", "@kbn/elastic-assistant-common", - "@kbn/logging" + "@kbn/logging", + "@kbn/react-kibana-context-render" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/serverless_search/public/application/connectors.tsx b/x-pack/plugins/serverless_search/public/application/connectors.tsx index 44d125117d2a84..0a00f6a5d68587 100644 --- a/x-pack/plugins/serverless_search/public/application/connectors.tsx +++ b/x-pack/plugins/serverless_search/public/application/connectors.tsx @@ -9,7 +9,7 @@ import { CoreStart } from '@kbn/core-lifecycle-browser'; import { I18nProvider } from '@kbn/i18n-react'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; -import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; @@ -28,7 +28,7 @@ export async function renderApp( const { ConnectorsRouter } = await import('./components/connectors_router'); ReactDOM.render( - + @@ -39,7 +39,7 @@ export async function renderApp( - , + , element ); return () => ReactDOM.unmountComponentAtNode(element); diff --git a/x-pack/plugins/serverless_search/public/application/elasticsearch.tsx b/x-pack/plugins/serverless_search/public/application/elasticsearch.tsx index 5413ff6361cf9c..8693ce18ace9d6 100644 --- a/x-pack/plugins/serverless_search/public/application/elasticsearch.tsx +++ b/x-pack/plugins/serverless_search/public/application/elasticsearch.tsx @@ -9,10 +9,9 @@ import React from 'react'; import ReactDOM from 'react-dom'; import { CoreStart } from '@kbn/core/public'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; -import { I18nProvider } from '@kbn/i18n-react'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; -import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; import { Route, Router, Routes } from '@kbn/shared-ux-router'; import { ServerlessSearchContext } from './hooks/use_kibana'; @@ -24,22 +23,20 @@ export async function renderApp( ) { const { ElasticsearchOverview } = await import('./components/overview'); ReactDOM.render( - + - - - - - - - - - + + + + + + + - , + , element ); return () => ReactDOM.unmountComponentAtNode(element); diff --git a/x-pack/plugins/serverless_search/public/test/test_utils.tsx b/x-pack/plugins/serverless_search/public/test/test_utils.tsx index ffc81b0b6daf6a..1012528cd3b360 100644 --- a/x-pack/plugins/serverless_search/public/test/test_utils.tsx +++ b/x-pack/plugins/serverless_search/public/test/test_utils.tsx @@ -8,7 +8,8 @@ import React, { ReactElement } from 'react'; import { render, RenderOptions } from '@testing-library/react'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; import { I18nProvider } from '@kbn/i18n-react'; import { coreMock } from '@kbn/core/public/mocks'; @@ -33,7 +34,7 @@ const queryClient = new QueryClient({ const AllTheProviders: React.FC = ({ children }) => { return ( - + {children} diff --git a/x-pack/plugins/serverless_search/tsconfig.json b/x-pack/plugins/serverless_search/tsconfig.json index d8762b4156ac2c..aa444365942733 100644 --- a/x-pack/plugins/serverless_search/tsconfig.json +++ b/x-pack/plugins/serverless_search/tsconfig.json @@ -47,5 +47,6 @@ "@kbn/discover-plugin", "@kbn/search-connectors-plugin", "@kbn/index-management", + "@kbn/react-kibana-context-render", ] } From 5d0714c36ac628023b3a5637bee51bbd1624a775 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:50:05 -0400 Subject: [PATCH 103/138] [Security Solution][Endpoint] enable `get-file` UI console command for SentinelOne agent types (#181162) ## Summary - Enables `get-file` response action via the Console UI for sentinelone - response action is gated behind feature flag `responseActionsSentinelOneGetFileEnabled` - Refactors the hook that opens the Response Console and removes `agentType` specific logic from it. Retrieval of console commands for a given agent type is now done in the `getEndpointConsoleCommands()` - Also refactored the `isResponseActionSupported()` and remove prior UI only implementation (not needed) - Un-skip isolate unit tests --- .../is_response_action_supported.ts | 69 ++------------ .../common/experimental_features.ts | 9 +- .../use_responder_action_data.ts | 7 +- .../get_file_action.tsx | 8 +- .../get_file_action.test.tsx | 74 +++++++++++++-- .../integration_tests/isolate_action.test.tsx | 3 +- .../lib/console_commands_definition.ts | 92 ++++++++++++++++--- .../hooks/use_with_show_responder.tsx | 48 ++-------- .../view/hooks/use_endpoint_action_items.tsx | 4 +- 9 files changed, 181 insertions(+), 133 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/service/response_actions/is_response_action_supported.ts b/x-pack/plugins/security_solution/common/endpoint/service/response_actions/is_response_action_supported.ts index 31dd195f00e03a..d197995de90d83 100644 --- a/x-pack/plugins/security_solution/common/endpoint/service/response_actions/is_response_action_supported.ts +++ b/x-pack/plugins/security_solution/common/endpoint/service/response_actions/is_response_action_supported.ts @@ -5,10 +5,7 @@ * 2.0. */ -import { getRbacControl } from './utils'; -import type { EndpointPrivileges } from '../../types'; import { - RESPONSE_ACTION_API_COMMAND_TO_CONSOLE_COMMAND_MAP, type ResponseActionAgentType, type ResponseActionsApiCommandNames, type ResponseActionType, @@ -19,65 +16,6 @@ type SupportMap = Record< Record> >; -/** @private */ -const getResponseActionsSupportMap = ({ - agentType, - actionName, - actionType, - privileges, -}: { - agentType: ResponseActionAgentType; - actionName: ResponseActionsApiCommandNames; - actionType: ResponseActionType; - privileges: EndpointPrivileges; -}): boolean => { - const commandName = RESPONSE_ACTION_API_COMMAND_TO_CONSOLE_COMMAND_MAP[actionName]; - const RESPONSE_ACTIONS_SUPPORT_MAP = { - [actionName]: { - automated: { - [agentType]: - agentType === 'endpoint' - ? getRbacControl({ - commandName, - privileges, - }) - : false, - }, - manual: { - [agentType]: - agentType === 'endpoint' - ? getRbacControl({ - commandName, - privileges, - }) - : actionName === 'isolate' || actionName === 'unisolate', - }, - }, - } as SupportMap; - return RESPONSE_ACTIONS_SUPPORT_MAP[actionName][actionType][agentType]; -}; - -/** - * Determine if a given response action is currently supported - * @param agentType - * @param actionName - * @param actionType - * @param privileges - */ -export const isResponseActionSupported = ( - agentType: ResponseActionAgentType, - actionName: ResponseActionsApiCommandNames, - actionType: ResponseActionType, - privileges: EndpointPrivileges -): boolean => { - return getResponseActionsSupportMap({ - privileges, - actionName, - actionType, - agentType, - }); -}; - /** @private */ const RESPONSE_ACTIONS_SUPPORT_MAP: SupportMap = { isolate: { @@ -162,7 +100,12 @@ const RESPONSE_ACTIONS_SUPPORT_MAP: SupportMap = { }, }; -// FIXME:PT reemove once this module is refactored. +/** + * Check if a given Response action is supported (implemented) for a given agent type and action type + * @param agentType + * @param actionName + * @param actionType + */ export const isActionSupportedByAgentType = ( agentType: ResponseActionAgentType, actionName: ResponseActionsApiCommandNames, diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 6004c15b222c59..edf11805a4de5b 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -80,8 +80,9 @@ export const allowedExperimentalValues = Object.freeze({ responseActionsSentinelOneV1Enabled: true, /** - * Enables use of SentinelOne response actions that complete asynchronously as well as support - * for more response actions. + * Enables use of SentinelOne response actions that complete asynchronously + * + * Release: v8.14.0 */ responseActionsSentinelOneV2Enabled: false, @@ -200,7 +201,9 @@ export const allowedExperimentalValues = Object.freeze({ sentinelOneDataInAnalyzerEnabled: true, /** - * Enables SentinelOne manual host manipulation actions + * Enables SentinelOne manual host isolation response actions directly through the connector + * sub-actions framework. + * v8.12.0 */ sentinelOneManualHostActionsEnabled: true, diff --git a/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts b/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts index fb20548271191e..5e17f3178d59c2 100644 --- a/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts +++ b/x-pack/plugins/security_solution/public/detections/components/endpoint_responder/use_responder_action_data.ts @@ -10,7 +10,10 @@ import type { TimelineEventsDetailsItem } from '@kbn/timelines-plugin/common'; import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; import { getSentinelOneAgentId } from '../../../common/utils/sentinelone_alert_check'; import type { ThirdPartyAgentInfo } from '../../../../common/types'; -import type { ResponseActionAgentType } from '../../../../common/endpoint/service/response_actions/constants'; +import type { + ResponseActionAgentType, + EndpointCapabilities, +} from '../../../../common/endpoint/service/response_actions/constants'; import { useGetEndpointDetails, useWithShowResponder } from '../../../management/hooks'; import { HostStatus } from '../../../../common/endpoint/types'; import { @@ -144,7 +147,7 @@ export const useResponderActionData = ({ showResponseActionsConsole({ agentId: hostInfo.metadata.agent.id, agentType: 'endpoint', - capabilities: hostInfo.metadata.Endpoint.capabilities ?? [], + capabilities: (hostInfo.metadata.Endpoint.capabilities as EndpointCapabilities[]) ?? [], hostName: hostInfo.metadata.host.name, }); } diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/get_file_action.tsx b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/get_file_action.tsx index 75c57f6ad43c4f..90e44c4a56ee2a 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/get_file_action.tsx +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/get_file_action.tsx @@ -25,9 +25,11 @@ export const GetFileActionResult = memo< const actionRequestBody = useMemo(() => { const endpointId = command.commandDefinition?.meta?.endpointId; const { path, comment } = command.args.args; + const agentType = command.commandDefinition?.meta?.agentType; return endpointId ? { + agent_type: agentType, endpoint_ids: [endpointId], comment: comment?.[0], parameters: { @@ -35,7 +37,11 @@ export const GetFileActionResult = memo< }, } : undefined; - }, [command.args.args, command.commandDefinition?.meta?.endpointId]); + }, [ + command.args.args, + command.commandDefinition?.meta?.agentType, + command.commandDefinition?.meta?.endpointId, + ]); const { result, actionDetails } = useConsoleActionSubmitter({ ResultComponent, diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/get_file_action.test.tsx b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/get_file_action.test.tsx index 9c6e49818daf67..06148aed5b483f 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/get_file_action.test.tsx +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/get_file_action.test.tsx @@ -14,6 +14,7 @@ import { ConsoleManagerTestComponent, getConsoleManagerMockRenderResultQueriesAndActions, } from '../../../console/components/console_manager/mocks'; +import type { GetEndpointConsoleCommandsOptions } from '../../lib/console_commands_definition'; import { getEndpointConsoleCommands } from '../../lib/console_commands_definition'; import React from 'react'; import { enterConsoleCommand } from '../../../console/mocks'; @@ -33,7 +34,6 @@ import type { HttpFetchOptionsWithPath } from '@kbn/core-http-browser'; import { endpointActionResponseCodes } from '../../lib/endpoint_action_response_codes'; jest.mock('../../../../../common/components/user_privileges'); -jest.mock('../../../../../common/experimental_features_service'); describe('When using get-file action from response actions console', () => { let render: ( @@ -45,13 +45,22 @@ describe('When using get-file action from response actions console', () => { typeof getConsoleManagerMockRenderResultQueriesAndActions >; let endpointPrivileges: EndpointPrivileges; + let getConsoleCommandsOptions: GetEndpointConsoleCommandsOptions; + let mockedContext: AppContextTestRender; beforeEach(() => { - const mockedContext = createAppRootMockRenderer(); + mockedContext = createAppRootMockRenderer(); apiMocks = responseActionsHttpMocks(mockedContext.coreStart.http); endpointPrivileges = { ...getEndpointAuthzInitialStateMock(), loading: false }; + getConsoleCommandsOptions = { + agentType: 'endpoint', + endpointAgentId: 'a.b.c', + endpointCapabilities: [...ENDPOINT_CAPABILITIES], + endpointPrivileges, + }; + render = async (capabilities: EndpointCapabilities[] = [...ENDPOINT_CAPABILITIES]) => { renderResult = mockedContext.render( { consoleProps: { 'data-test-subj': 'test', commands: getEndpointConsoleCommands({ - agentType: 'endpoint', - endpointAgentId: 'a.b.c', - endpointCapabilities: [...capabilities], - endpointPrivileges, + ...getConsoleCommandsOptions, + endpointCapabilities: capabilities, }), }, }; @@ -123,7 +130,7 @@ describe('When using get-file action from response actions console', () => { await waitFor(() => { expect(apiMocks.responseProvider.getFile).toHaveBeenCalledWith({ - body: '{"endpoint_ids":["a.b.c"],"parameters":{"path":"one/two"}}', + body: '{"agent_type":"endpoint","endpoint_ids":["a.b.c"],"parameters":{"path":"one/two"}}', path: GET_FILE_ROUTE, version: '2023-10-31', }); @@ -204,4 +211,57 @@ describe('When using get-file action from response actions console', () => { ); }); }); + + describe('And agent type is SentinelOne', () => { + beforeEach(() => { + getConsoleCommandsOptions.agentType = 'sentinel_one'; + mockedContext.setExperimentalFlag({ + responseActionsSentinelOneGetFileEnabled: true, + }); + }); + + it('should display error if feature flag is not enabled', async () => { + mockedContext.setExperimentalFlag({ + responseActionsSentinelOneGetFileEnabled: false, + }); + await render(); + enterConsoleCommand(renderResult, 'get-file --path="one/two"'); + + expect(renderResult.getByTestId('test-validationError-message').textContent).toEqual( + UPGRADE_AGENT_FOR_RESPONDER('sentinel_one', 'get-file') + ); + }); + + it('should call API with `agent_type` set to `sentinel_one`', async () => { + await render(); + enterConsoleCommand(renderResult, 'get-file --path="one/two"'); + + await waitFor(() => { + expect(apiMocks.responseProvider.getFile).toHaveBeenCalledWith({ + body: '{"agent_type":"sentinel_one","endpoint_ids":["a.b.c"],"parameters":{"path":"one/two"}}', + path: GET_FILE_ROUTE, + version: '2023-10-31', + }); + }); + }); + + it('should not look at `capabilities` to determine compatibility', async () => { + await render([]); + enterConsoleCommand(renderResult, 'get-file --path="one/two"'); + + await waitFor(() => { + expect(apiMocks.responseProvider.getFile).toHaveBeenCalled(); + }); + expect(renderResult.queryByTestId('test-validationError-message')).toBeNull(); + }); + + it('should display pending message', async () => { + await render(); + enterConsoleCommand(renderResult, 'get-file --path="one/two"'); + + await waitFor(() => { + expect(renderResult.getByTestId('getFile-pending')); + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/isolate_action.test.tsx b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/isolate_action.test.tsx index 77e70fc14180ce..f5a20be31580a1 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/isolate_action.test.tsx +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/command_render_components/integration_tests/isolate_action.test.tsx @@ -24,8 +24,7 @@ import { UPGRADE_AGENT_FOR_RESPONDER } from '../../../../../common/translations' jest.mock('../../../../../common/experimental_features_service'); -// FLAKY https://github.com/elastic/kibana/issues/145363 -describe.skip('When using isolate action from response actions console', () => { +describe('When using isolate action from response actions console', () => { let render: ( capabilities?: EndpointCapabilities[] ) => Promise>; diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/lib/console_commands_definition.ts b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/lib/console_commands_definition.ts index 238efec7542dca..61fb75cb524502 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_responder/lib/console_commands_definition.ts +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_responder/lib/console_commands_definition.ts @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import { isActionSupportedByAgentType } from '../../../../../common/endpoint/service/response_actions/is_response_action_supported'; import { getRbacControl } from '../../../../../common/endpoint/service/response_actions/utils'; import { UploadActionResult } from '../command_render_components/upload_action'; import { ArgumentFileSelector } from '../../console_argument_selectors'; @@ -16,7 +17,10 @@ import type { EndpointCapabilities, ResponseActionAgentType, } from '../../../../../common/endpoint/service/response_actions/constants'; -import { RESPONSE_CONSOLE_ACTION_COMMANDS_TO_ENDPOINT_CAPABILITY } from '../../../../../common/endpoint/service/response_actions/constants'; +import { + RESPONSE_CONSOLE_ACTION_COMMANDS_TO_ENDPOINT_CAPABILITY, + RESPONSE_CONSOLE_COMMAND_TO_API_COMMAND_MAP, +} from '../../../../../common/endpoint/service/response_actions/constants'; import { GetFileActionResult } from '../command_render_components/get_file_action'; import type { Command, CommandDefinition } from '../../console'; import { IsolateActionResult } from '../command_render_components/isolate_action'; @@ -83,14 +87,18 @@ const capabilitiesAndPrivilegesValidator = ( const responderCapability = RESPONSE_CONSOLE_ACTION_COMMANDS_TO_ENDPOINT_CAPABILITY[commandName]; let errorMessage = ''; - if (!responderCapability) { - errorMessage = errorMessage.concat(UPGRADE_AGENT_FOR_RESPONDER(agentType, commandName)); - } - if (responderCapability) { - if (!agentCapabilities.includes(responderCapability)) { + + // We only validate Agent capabilities for the command for Endpoint agents + if (agentType === 'endpoint') { + if (!responderCapability) { + errorMessage = errorMessage.concat(UPGRADE_AGENT_FOR_RESPONDER(agentType, commandName)); + } + + if (responderCapability && !agentCapabilities.includes(responderCapability)) { errorMessage = errorMessage.concat(UPGRADE_AGENT_FOR_RESPONDER(agentType, commandName)); } } + if (!getRbacControl({ commandName, privileges })) { errorMessage = errorMessage.concat(INSUFFICIENT_PRIVILEGES_FOR_COMMAND); } @@ -127,27 +135,36 @@ const COMMENT_ARG_ABOUT = i18n.translate( { defaultMessage: 'A comment to go along with the action' } ); +export interface GetEndpointConsoleCommandsOptions { + endpointAgentId: string; + agentType: ResponseActionAgentType; + endpointCapabilities: ImmutableArray; + endpointPrivileges: EndpointPrivileges; +} + export const getEndpointConsoleCommands = ({ endpointAgentId, agentType, endpointCapabilities, endpointPrivileges, -}: { - endpointAgentId: string; - agentType: ResponseActionAgentType; - endpointCapabilities: ImmutableArray; - endpointPrivileges: EndpointPrivileges; -}): CommandDefinition[] => { +}: GetEndpointConsoleCommandsOptions): CommandDefinition[] => { const featureFlags = ExperimentalFeaturesService.get(); const isUploadEnabled = featureFlags.responseActionUploadEnabled; const doesEndpointSupportCommand = (commandName: ConsoleResponseActionCommands) => { + // Agent capabilities is only validated for Endpoint agent types + if (agentType !== 'endpoint') { + return true; + } + const responderCapability = RESPONSE_CONSOLE_ACTION_COMMANDS_TO_ENDPOINT_CAPABILITY[commandName]; + if (responderCapability) { return endpointCapabilities.includes(responderCapability); } + return false; }; @@ -484,5 +501,54 @@ export const getEndpointConsoleCommands = ({ }); } - return consoleCommands; + switch (agentType) { + case 'sentinel_one': + return adjustCommandsForSentinelOne({ commandList: consoleCommands }); + default: + // agentType === endpoint: just returns the defined command list + return consoleCommands; + } +}; + +/** @private */ +const adjustCommandsForSentinelOne = ({ + commandList, +}: { + commandList: CommandDefinition[]; +}): CommandDefinition[] => { + const featureFlags = ExperimentalFeaturesService.get(); + const isHostIsolationEnabled = featureFlags.responseActionsSentinelOneV1Enabled; + const isGetFileFeatureEnabled = featureFlags.responseActionsSentinelOneGetFileEnabled; + + const disableCommand = (command: CommandDefinition) => { + command.helpDisabled = true; + command.helpHidden = true; + command.validate = () => + UPGRADE_AGENT_FOR_RESPONDER('sentinel_one', command.name as ConsoleResponseActionCommands); + }; + + return commandList.map((command) => { + const agentSupportsResponseAction = + command.name === 'status' + ? false + : isActionSupportedByAgentType( + 'sentinel_one', + RESPONSE_CONSOLE_COMMAND_TO_API_COMMAND_MAP[ + command.name as ConsoleResponseActionCommands + ], + 'manual' + ); + + // If command is not supported by SentinelOne - disable it + if ( + !agentSupportsResponseAction || + (command.name === 'get-file' && !isGetFileFeatureEnabled) || + (command.name === 'isolate' && !isHostIsolationEnabled) || + (command.name === 'release' && !isHostIsolationEnabled) + ) { + disableCommand(command); + } + + return command; + }); }; diff --git a/x-pack/plugins/security_solution/public/management/hooks/use_with_show_responder.tsx b/x-pack/plugins/security_solution/public/management/hooks/use_with_show_responder.tsx index c36b02d90ccf0f..15186ddc486f31 100644 --- a/x-pack/plugins/security_solution/public/management/hooks/use_with_show_responder.tsx +++ b/x-pack/plugins/security_solution/public/management/hooks/use_with_show_responder.tsx @@ -7,19 +7,11 @@ import React, { useCallback } from 'react'; import { EuiBetaBadge, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { - TECHNICAL_PREVIEW, - TECHNICAL_PREVIEW_TOOLTIP, - UPGRADE_AGENT_FOR_RESPONDER, -} from '../../common/translations'; +import { TECHNICAL_PREVIEW, TECHNICAL_PREVIEW_TOOLTIP } from '../../common/translations'; import { useLicense } from '../../common/hooks/use_license'; -import type { ImmutableArray } from '../../../common/endpoint/types'; -import { - type ConsoleResponseActionCommands, - RESPONSE_CONSOLE_COMMAND_TO_API_COMMAND_MAP, - type ResponseActionAgentType, -} from '../../../common/endpoint/service/response_actions/constants'; -import { isResponseActionSupported } from '../../../common/endpoint/service/response_actions/is_response_action_supported'; +import type { MaybeImmutable } from '../../../common/endpoint/types'; +import type { EndpointCapabilities } from '../../../common/endpoint/service/response_actions/constants'; +import { type ResponseActionAgentType } from '../../../common/endpoint/service/response_actions/constants'; import { HeaderSentinelOneInfo } from '../components/endpoint_responder/components/header_info/sentinel_one/header_sentinel_one_info'; import { useUserPrivileges } from '../../common/components/user_privileges'; @@ -39,16 +31,16 @@ type ShowResponseActionsConsole = (props: ResponderInfoProps) => void; export interface BasicConsoleProps { agentId: string; hostName: string; + /** Required for Endpoint agents. */ + capabilities: MaybeImmutable; } type ResponderInfoProps = | (BasicConsoleProps & { agentType: Extract; - capabilities: ImmutableArray; }) | (BasicConsoleProps & { agentType: Exclude; - capabilities: ImmutableArray; platform: string; }); @@ -85,33 +77,6 @@ export const useWithShowResponder = (): ShowResponseActionsConsole => { endpointAgentId: agentId, endpointCapabilities: capabilities, endpointPrivileges, - }).map((command) => { - if (command.name !== 'status') { - return { - ...command, - helpHidden: !isResponseActionSupported( - agentType, - RESPONSE_CONSOLE_COMMAND_TO_API_COMMAND_MAP[ - command.name as ConsoleResponseActionCommands - ], - 'manual', - endpointPrivileges - ), - }; - } else if (agentType !== 'endpoint') { - // do not show 'status' for non-endpoint agents - return { - ...command, - helpHidden: true, - validate: () => { - return UPGRADE_AGENT_FOR_RESPONDER( - agentType, - command.name as ConsoleResponseActionCommands - ); - }, - }; - } - return command; }), 'data-test-subj': `${agentType}ResponseActionsConsole`, storagePrefix: 'xpack.securitySolution.Responder', @@ -138,6 +103,7 @@ export const useWithShowResponder = (): ShowResponseActionsConsole => { meta: { agentId, hostName, + capabilities, }, consoleProps, PageTitleComponent: () => { diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx index 6f18c60d4dc6b1..1bd0c3dff62ff0 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/hooks/use_endpoint_action_items.tsx @@ -8,6 +8,7 @@ import React, { useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { pagePathGetters } from '@kbn/fleet-plugin/public'; +import type { EndpointCapabilities } from '../../../../../../common/endpoint/service/response_actions/constants'; import { useUserPrivileges } from '../../../../../common/components/user_privileges'; import { useWithShowResponder } from '../../../../hooks'; import { APP_UI_ID } from '../../../../../../common/constants'; @@ -130,7 +131,8 @@ export const useEndpointActionItems = ( showEndpointResponseActionsConsole({ agentId: endpointMetadata.agent.id, agentType: 'endpoint', - capabilities: endpointMetadata.Endpoint.capabilities ?? [], + capabilities: + (endpointMetadata.Endpoint.capabilities as EndpointCapabilities[]) ?? [], hostName: endpointMetadata.host.name, }); }, From b64b72a8beca14e62f59da006f2fc1b8dd804e5a Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Thu, 25 Apr 2024 15:59:12 +0200 Subject: [PATCH 104/138] [CI] fix typo in deployment purge criteria (#181704) ## Summary This was causing `elasticsearch` deployments to vanish before their time is due. cc: @pgayvallet --- .buildkite/scripts/steps/cloud/purge_projects.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/cloud/purge_projects.ts b/.buildkite/scripts/steps/cloud/purge_projects.ts index a8a83266a826e0..84083417f62678 100644 --- a/.buildkite/scripts/steps/cloud/purge_projects.ts +++ b/.buildkite/scripts/steps/cloud/purge_projects.ts @@ -88,7 +88,7 @@ async function purgeProjects() { } else if ( !Boolean( pullRequest.labels.filter((label: any) => - /^ci:project-deploy-(elasticearch|security|observability)$/.test(label.name) + /^ci:project-deploy-(elasticsearch|security|observability)$/.test(label.name) ).length ) ) { From 7c447adf4cd62f33382c550046e546387111a838 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 25 Apr 2024 17:11:03 +0200 Subject: [PATCH 105/138] [Security Solution][Serverless] Fix project features url (#181608) ## Summary Fixes the double slash (`//`) in the URL to manage the serverless project features from the Get Started page: before: `https://console.qa.cld.elstc.co/projects//security/:id` before After: `https://console.qa.cld.elstc.co/projects/security/:id` after Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/navigation/util.test.ts | 4 ++-- .../security_solution_serverless/public/navigation/util.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution_serverless/public/navigation/util.test.ts b/x-pack/plugins/security_solution_serverless/public/navigation/util.test.ts index fd03f6f2ecdf30..6ea4a935b99987 100644 --- a/x-pack/plugins/security_solution_serverless/public/navigation/util.test.ts +++ b/x-pack/plugins/security_solution_serverless/public/navigation/util.test.ts @@ -12,7 +12,7 @@ const cloud = { serverless: { projectId: '1234', }, - projectsUrl: 'https://cloud.elastic.co/projects', + projectsUrl: 'https://cloud.elastic.co/projects/', } as CloudStart; describe('util', () => { @@ -29,7 +29,7 @@ describe('util', () => { it('should return the correct url', () => { expect(getProjectFeaturesUrl(cloud)).toBe( - `${cloud.projectsUrl}/security/${cloud.serverless?.projectId}?open=securityProjectFeatures` + `${cloud.projectsUrl}security/${cloud.serverless?.projectId}?open=securityProjectFeatures` ); }); }); diff --git a/x-pack/plugins/security_solution_serverless/public/navigation/util.ts b/x-pack/plugins/security_solution_serverless/public/navigation/util.ts index d57b4f7e1a4abe..ca7e1e07d6922b 100644 --- a/x-pack/plugins/security_solution_serverless/public/navigation/util.ts +++ b/x-pack/plugins/security_solution_serverless/public/navigation/util.ts @@ -20,7 +20,7 @@ export const getProjectFeaturesUrl = (cloud: CloudStart): string | undefined => if (!projectsBaseUrl || !projectId) { return undefined; } - return `${projectsBaseUrl}/${SECURITY_PROJECT_TYPE}/${projectId}?open=securityProjectFeatures`; + return `${projectsBaseUrl}${SECURITY_PROJECT_TYPE}/${projectId}?open=securityProjectFeatures`; }; export const getCloudUrl: GetCloudUrl = (cloudUrlKey, cloud) => { From 8e758d936bd15784a7a0819c45699b55bf57769b Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Thu, 25 Apr 2024 17:19:06 +0200 Subject: [PATCH 106/138] [Alert details page][Log threshold] Fix alert number annotation on the history chart (#181702) Fixes #175203 ### Summary |Before|After| |---|---| |![image](https://github.com/elastic/kibana/assets/12370520/ba83f309-c7c5-4d2d-a8de-832e80bc6eb5)|![image](https://github.com/elastic/kibana/assets/12370520/8c0a5b3a-a420-47fe-be9f-bf184d64809c)| --- .../expression_editor/criterion_preview_chart.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx b/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx index 47c826178bc9b0..af3bbb0ae92732 100644 --- a/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx +++ b/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx @@ -333,7 +333,11 @@ const CriterionPreviewChart: React.FC = ({ tickFormat={yAxisFormatter} domain={chartDomain} /> - + From b4e0575882fef55a1d54e34a463c3d139e8b7f31 Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Thu, 25 Apr 2024 17:30:52 +0200 Subject: [PATCH 107/138] [ES|QL] Small refactoring to ensure that the localstorage limit will always be respected (#181415) ## Summary Make the guard of 20 max queries in the local storage more robust. This is just a refactoring of the implementation. In case anything goes wrong, it makes sure that the queries will always be the maximum allowed. --- .../src/history_local_storage.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/kbn-text-based-editor/src/history_local_storage.ts b/packages/kbn-text-based-editor/src/history_local_storage.ts index dfe3b2e05b17c7..b6bbdd7d5896a8 100644 --- a/packages/kbn-text-based-editor/src/history_local_storage.ts +++ b/packages/kbn-text-based-editor/src/history_local_storage.ts @@ -102,16 +102,17 @@ export const updateCachedQueries = ( ); let allQueries = [...queriesToStore, ...newQueries]; - if (allQueries.length === maxQueriesAllowed + 1) { + if (allQueries.length >= maxQueriesAllowed + 1) { const sortedByDate = allQueries.sort((a, b) => sortDates(b?.startDateMilliseconds, a?.startDateMilliseconds) ); - // delete the last element - const toBeDeletedQuery = sortedByDate[maxQueriesAllowed]; - cachedQueries.delete(toBeDeletedQuery.queryString); - allQueries = allQueries.filter((q) => { - return q.queryString !== toBeDeletedQuery.queryString; + // queries to store in the localstorage + allQueries = sortedByDate.slice(0, maxQueriesAllowed); + // clear and reset the queries in the cache + cachedQueries.clear(); + allQueries.forEach((queryItem) => { + cachedQueries.set(queryItem.queryString, queryItem); }); } localStorage.setItem(QUERY_HISTORY_ITEM_KEY, JSON.stringify(allQueries)); From 1d150fb22f0390c185bd865600729469a13fde57 Mon Sep 17 00:00:00 2001 From: acrewdson Date: Thu, 25 Apr 2024 08:37:36 -0700 Subject: [PATCH 108/138] Use more idiomatic phrasing in connectors delete modal (#181627) Updates the connectors 'delete' modal text to use a more typical word order. --- .../components/connectors/delete_connector_modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connectors/delete_connector_modal.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connectors/delete_connector_modal.tsx index 84b992c89c86cb..512b0bd384697f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connectors/delete_connector_modal.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connectors/delete_connector_modal.tsx @@ -170,7 +170,7 @@ export const DeleteConnectorModal: React.FC = ({ isCr id="delete-related-index" label={i18n.translate( 'xpack.enterpriseSearch.deleteConnectorModal.euiCheckbox.deleteAlsoRelatedIndexLabel', - { defaultMessage: 'Delete also related index' } + { defaultMessage: 'Also delete related index' } )} checked={shouldDeleteIndex} onChange={() => setShouldDeleteIndex(!shouldDeleteIndex)} From d10ffc5acc2c4fd972f6d385f1dcd6a824c9500c Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 25 Apr 2024 16:48:04 +0100 Subject: [PATCH 109/138] [ML] Adding ML feature privileges tooltip (#181595) Adds a tooltip to the machine learning feature to inform users that an ML all privilege also grants some additional saved object privileges. ![image](https://github.com/elastic/kibana/assets/22172091/73023a51-91b9-4eb9-8889-5d9d9fad1be7) --- x-pack/plugins/ml/server/plugin.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index b8cc406eaeadf6..ee36bd73828434 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -133,6 +133,10 @@ export class MlServerPlugin category: DEFAULT_APP_CATEGORIES.kibana, app: [PLUGIN_ID, 'kibana'], catalogue: [PLUGIN_ID, `${PLUGIN_ID}_file_data_visualizer`], + privilegesTooltip: i18n.translate('xpack.ml.featureRegistry.privilegesTooltip', { + defaultMessage: + 'Granting All or Read feature privilege for Machine Learning will also grant the equivalent feature privileges to certain types of Kibana saved objects, namely index patterns, dashboards, saved searches and visualizations as well as machine learning job, trained model and module saved objects.', + }), management: { insightsAndAlerting: ['jobsListLink', 'triggersActions'], }, From aa09f35d13076ef32c30129af7adc0ba8fedfb33 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 25 Apr 2024 17:51:25 +0200 Subject: [PATCH 110/138] [kbn-test] add codeOwners in junit report (#181711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Related to #180802 This PR adds `codeOwners` attribute in FTR JUnit report to the failed test case: ``` ``` QAF will parse JUnit report to get failures and owner, that later can be used for Slack notification Note for reviewers: we are aware that the new attribute is not following JUnit validation schema, but it seems like the best option since we can't add property on `testcase` element --- .../src/mocha/junit_report_generation.js | 21 +++++++++++++++---- .../src/mocha/junit_report_generation.test.js | 21 +++++++++---------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/packages/kbn-test/src/mocha/junit_report_generation.js b/packages/kbn-test/src/mocha/junit_report_generation.js index 001fe79a380612..4b35fba4fb1e62 100644 --- a/packages/kbn-test/src/mocha/junit_report_generation.js +++ b/packages/kbn-test/src/mocha/junit_report_generation.js @@ -7,6 +7,7 @@ */ import { REPO_ROOT } from '@kbn/repo-info'; +import { getCodeOwnersForFile, getPathsWithOwnersReversed } from '@kbn/code-owners'; import { dirname, relative } from 'path'; import { writeFileSync, mkdirSync } from 'fs'; import { inspect } from 'util'; @@ -91,6 +92,9 @@ export function setupJUnitReportGeneration(runner, options = {}) { .filter((node) => node.pending || !results.find((result) => result.node === node)) .map((node) => ({ skipped: true, node })); + // cache codeowners for quicker lookup + const reversedCodeowners = getPathsWithOwnersReversed(); + const builder = xmlBuilder.create( 'testsuites', { encoding: 'utf-8' }, @@ -108,17 +112,26 @@ export function setupJUnitReportGeneration(runner, options = {}) { 'metadata-json': JSON.stringify(metadata ?? {}), }); - function addTestcaseEl(node) { - return testsuitesEl.ele('testcase', { + function addTestcaseEl(node, failed) { + const attrs = { name: getFullTitle(node), classname: `${reportName}.${getPath(node).replace(/\./g, '·')}`, time: getDuration(node), 'metadata-json': JSON.stringify(getTestMetadata(node) || {}), - }); + }; + + // adding code owners only for the failed test case + if (failed) { + const testCaseRelativePath = getPath(node); + const owners = getCodeOwnersForFile(testCaseRelativePath, reversedCodeowners); + attrs.owners = owners || ''; // empty string when no codeowners are defined + } + + return testsuitesEl.ele('testcase', attrs); } [...results, ...skippedResults].forEach((result) => { - const el = addTestcaseEl(result.node); + const el = addTestcaseEl(result.node, result.failed); if (result.failed) { el.ele('system-out').dat(escapeCdata(getSnapshotOfRunnableLogs(result.node) || '')); diff --git a/packages/kbn-test/src/mocha/junit_report_generation.test.js b/packages/kbn-test/src/mocha/junit_report_generation.test.js index ac23d91390ed97..b6bc2e951d1df9 100644 --- a/packages/kbn-test/src/mocha/junit_report_generation.test.js +++ b/packages/kbn-test/src/mocha/junit_report_generation.test.js @@ -54,17 +54,14 @@ describe('dev/mocha/junit report generation', () => { const [testsuite] = report.testsuites.testsuite; expect(testsuite.$.time).toMatch(DURATION_REGEX); expect(testsuite.$.timestamp).toMatch(ISO_DATE_SEC_REGEX); - expect(testsuite).toEqual({ - $: { - failures: '2', - name: 'test', - skipped: '1', - tests: '4', - 'metadata-json': '{}', - time: testsuite.$.time, - timestamp: testsuite.$.timestamp, - }, - testcase: testsuite.testcase, + expect(testsuite.$).toEqual({ + failures: '2', + name: 'test', + skipped: '1', + tests: '4', + 'metadata-json': '{}', + time: testsuite.$.time, + timestamp: testsuite.$.timestamp, }); // there are actually only three tests, but since the hook failed @@ -94,6 +91,7 @@ describe('dev/mocha/junit report generation', () => { name: 'SUITE fails', time: testFail.$.time, 'metadata-json': '{}', + owners: '', }, 'system-out': testFail['system-out'], failure: [testFail.failure[0]], @@ -108,6 +106,7 @@ describe('dev/mocha/junit report generation', () => { name: 'SUITE SUB_SUITE "before each" hook: fail hook for "never runs"', time: beforeEachFail.$.time, 'metadata-json': '{}', + owners: '', }, 'system-out': testFail['system-out'], failure: [beforeEachFail.failure[0]], From 15c6a36eeb735af5aef8bb5564efbd4f28f7bc47 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Thu, 25 Apr 2024 09:05:29 -0700 Subject: [PATCH 111/138] [Reporting] Remove usage of deprecated React rendering utilities (#180759) ## Summary Partially addresses https://github.com/elastic/kibana-team/issues/805 Follows https://github.com/elastic/kibana/pull/180516 These changes come up from searching in the code and finding where certain kinds of deprecated AppEx-SharedUX modules are imported. **Reviewers: Please interact with critical paths through the UI components touched in this PR, ESPECIALLY in terms of testing dark mode and i18n.** This focuses on code within Reporting. image Note: this also makes inclusion of `i18n` and `analytics` dependencies consistent. Analytics is an optional dependency for the SharedUX modules, which wrap `KibanaErrorBoundaryProvider` and is designed to capture telemetry about errors that are caught in the error boundary. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../panel_actions/get_csv_panel_action.tsx | 17 ++- packages/kbn-reporting/public/share/index.ts | 2 +- .../public/share/share_context_menu/index.ts | 33 +++-- .../register_csv_modal_reporting.tsx | 13 +- .../register_csv_reporting.tsx | 8 +- .../register_pdf_png_modal_reporting.tsx | 33 ++--- .../register_pdf_png_reporting.tsx | 12 +- .../reporting_panel_content.test.tsx | 16 +-- .../reporting_panel_content.tsx | 127 +++++++++--------- .../screen_capture_panel_content.test.tsx | 32 ++--- .../share/shared/get_shared_components.tsx | 42 +++--- packages/kbn-reporting/public/tsconfig.json | 3 + packages/kbn-reporting/public/types.ts | 16 +++ .../public/lib/stream_handler.test.ts | 76 ++--------- .../reporting/public/lib/stream_handler.ts | 43 +++--- .../management/mount_management_section.tsx | 39 +++--- x-pack/plugins/reporting/public/mocks.ts | 3 +- .../public/notifier/general_error.tsx | 12 +- .../reporting/public/notifier/job_failure.tsx | 12 +- .../reporting/public/notifier/job_success.tsx | 10 +- .../reporting/public/notifier/job_warning.tsx | 10 +- .../public/notifier/job_warning_formulas.tsx | 10 +- .../public/notifier/job_warning_max_size.tsx | 10 +- x-pack/plugins/reporting/public/plugin.ts | 89 ++++++------ x-pack/plugins/reporting/public/types.ts | 21 +++ x-pack/plugins/reporting/tsconfig.json | 3 +- 26 files changed, 318 insertions(+), 374 deletions(-) diff --git a/packages/kbn-reporting/get_csv_panel_actions/panel_actions/get_csv_panel_action.tsx b/packages/kbn-reporting/get_csv_panel_actions/panel_actions/get_csv_panel_action.tsx index cd34d64a8429aa..bcf3f2af51956a 100644 --- a/packages/kbn-reporting/get_csv_panel_actions/panel_actions/get_csv_panel_action.tsx +++ b/packages/kbn-reporting/get_csv_panel_actions/panel_actions/get_csv_panel_action.tsx @@ -48,11 +48,26 @@ export interface PanelActionDependencies { licensing: LicensingPluginStart; } +type StartServices = [ + Pick< + CoreStart, + // required for modules that render React + | 'analytics' + | 'i18n' + | 'theme' + // used extensively in Reporting share panel action + | 'application' + | 'uiSettings' + >, + PanelActionDependencies, + unknown +]; + interface Params { apiClient: ReportingAPIClient; csvConfig: ClientConfigType['csv']; core: CoreSetup; - startServices$: Observable<[CoreStart, PanelActionDependencies, unknown]>; + startServices$: Observable; usesUiCapabilities: boolean; } diff --git a/packages/kbn-reporting/public/share/index.ts b/packages/kbn-reporting/public/share/index.ts index 7c7b6819afc071..b2587965858d82 100644 --- a/packages/kbn-reporting/public/share/index.ts +++ b/packages/kbn-reporting/public/share/index.ts @@ -12,4 +12,4 @@ export { reportingScreenshotShareProvider } from './share_context_menu/register_ export { reportingCsvShareProvider } from './share_context_menu/register_csv_reporting'; export { reportingCsvShareProvider as reportingCsvShareModalProvider } from './share_context_menu/register_csv_modal_reporting'; export type { ReportingPublicComponents } from './shared/get_shared_components'; -export type { JobParamsProviderOptions } from './share_context_menu'; +export type { JobParamsProviderOptions, StartServices } from './share_context_menu'; diff --git a/packages/kbn-reporting/public/share/share_context_menu/index.ts b/packages/kbn-reporting/public/share/share_context_menu/index.ts index 1259aee0008ad6..c27ec3f38c68c5 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/index.ts +++ b/packages/kbn-reporting/public/share/share_context_menu/index.ts @@ -6,35 +6,42 @@ * Side Public License, v 1. */ -import type { - ApplicationStart, - I18nStart, - IUiSettingsClient, - ThemeServiceSetup, - ToastsSetup, -} from '@kbn/core/public'; +import * as Rx from 'rxjs'; + +import type { ApplicationStart, CoreStart } from '@kbn/core/public'; import { ILicense } from '@kbn/licensing-plugin/public'; import type { LayoutParams } from '@kbn/screenshotting-plugin/common'; + import type { ReportingAPIClient } from '../../reporting_api_client'; +export type StartServices = [ + Pick< + CoreStart, + // required for modules that render React + | 'analytics' + | 'i18n' + | 'theme' + // used extensively in Reporting share context menus and modal + | 'notifications' + >, + unknown, + unknown +]; + export interface ExportModalShareOpts { apiClient: ReportingAPIClient; - uiSettings: IUiSettingsClient; usesUiCapabilities: boolean; license: ILicense; application: ApplicationStart; - theme: ThemeServiceSetup; - i18n: I18nStart; + startServices$: Rx.Observable; } export interface ExportPanelShareOpts { apiClient: ReportingAPIClient; - toasts: ToastsSetup; - uiSettings: IUiSettingsClient; usesUiCapabilities: boolean; license: ILicense; application: ApplicationStart; - theme: ThemeServiceSetup; + startServices$: Rx.Observable; } export interface ReportingSharingData { diff --git a/packages/kbn-reporting/public/share/share_context_menu/register_csv_modal_reporting.tsx b/packages/kbn-reporting/public/share/share_context_menu/register_csv_modal_reporting.tsx index 7525c714de7b25..70225a3033773e 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/register_csv_modal_reporting.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/register_csv_modal_reporting.tsx @@ -7,14 +7,15 @@ */ import { i18n } from '@kbn/i18n'; -import React from 'react'; import { toMountPoint } from '@kbn/react-kibana-mount'; +import React from 'react'; +import { firstValueFrom } from 'rxjs'; import { CSV_JOB_TYPE, CSV_JOB_TYPE_V2 } from '@kbn/reporting-export-types-csv-common'; import type { SearchSourceFields } from '@kbn/data-plugin/common'; -import { ShareContext, ShareMenuItem } from '@kbn/share-plugin/public'; import { FormattedMessage, InjectedIntl } from '@kbn/i18n-react'; +import { ShareContext, ShareMenuItem } from '@kbn/share-plugin/public'; import type { ExportModalShareOpts } from '.'; import { checkLicense } from '../..'; @@ -23,8 +24,7 @@ export const reportingCsvShareProvider = ({ application, license, usesUiCapabilities, - i18n: i18nStart, - theme, + startServices$, }: ExportModalShareOpts) => { const getShareMenuItems = ({ objectType, sharingData, toasts }: ShareContext) => { if ('search' !== objectType) { @@ -86,7 +86,8 @@ export const reportingCsvShareProvider = ({ const decoratedJobParams = apiClient.getDecoratedJobParams(getJobParams()); return apiClient .createReportingJob(reportType, decoratedJobParams) - .then(() => { + .then(() => firstValueFrom(startServices$)) + .then(([startServices]) => { toasts.addSuccess({ title: intl.formatMessage( { @@ -110,7 +111,7 @@ export const reportingCsvShareProvider = ({ ), }} />, - { theme, i18n: i18nStart } + startServices ), 'data-test-subj': 'queueReportSuccess', }); diff --git a/packages/kbn-reporting/public/share/share_context_menu/register_csv_reporting.tsx b/packages/kbn-reporting/public/share/share_context_menu/register_csv_reporting.tsx index 58c380a2201e49..5144d32bc48cda 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/register_csv_reporting.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/register_csv_reporting.tsx @@ -19,12 +19,10 @@ import { ReportingPanelContent } from './reporting_panel_content_lazy'; export const reportingCsvShareProvider = ({ apiClient, - toasts, - uiSettings, application, license, usesUiCapabilities, - theme, + startServices$, }: ExportPanelShareOpts): ShareMenuProvider => { const getShareMenuItems = ({ objectType, objectId, sharingData, onClose }: ShareContext) => { if ('search' !== objectType) { @@ -104,14 +102,12 @@ export const reportingCsvShareProvider = ({ ), }, diff --git a/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_modal_reporting.tsx b/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_modal_reporting.tsx index 1761b8df45878b..621ab6fc5a0d34 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_modal_reporting.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_modal_reporting.tsx @@ -7,17 +7,18 @@ */ import { i18n } from '@kbn/i18n'; -import React from 'react'; -import { ShareContext, ShareMenuItem, ShareMenuProvider } from '@kbn/share-plugin/public'; import { FormattedMessage, InjectedIntl } from '@kbn/i18n-react'; import { toMountPoint } from '@kbn/react-kibana-mount'; -import { checkLicense } from '../../license_check'; +import { ShareContext, ShareMenuItem, ShareMenuProvider } from '@kbn/share-plugin/public'; +import React from 'react'; +import { firstValueFrom } from 'rxjs'; import { ExportModalShareOpts, ExportPanelShareOpts, JobParamsProviderOptions, ReportingSharingData, } from '.'; +import { checkLicense } from '../../license_check'; import { ScreenCapturePanelContent } from './screen_capture_panel_content_lazy'; const getJobParams = (opts: JobParamsProviderOptions, type: 'pngV2' | 'printablePdfV2') => () => { @@ -45,12 +46,10 @@ const getJobParams = (opts: JobParamsProviderOptions, type: 'pngV2' | 'printable */ export const reportingScreenshotShareProvider = ({ apiClient, - toasts, - uiSettings, license, application, usesUiCapabilities, - theme, + startServices$, }: ExportPanelShareOpts): ShareMenuProvider => { const getShareMenuItems = ({ objectType, @@ -136,15 +135,13 @@ export const reportingScreenshotShareProvider = ({ content: ( ), }, @@ -169,8 +166,7 @@ export const reportingScreenshotShareProvider = ({ content: ( ), }, @@ -200,8 +195,7 @@ export const reportingExportModalProvider = ({ license, application, usesUiCapabilities, - theme, - i18n: i18nStart, + startServices$, }: ExportModalShareOpts): ShareMenuProvider => { const getShareMenuItems = ({ objectType, @@ -294,7 +288,8 @@ export const reportingExportModalProvider = ({ return apiClient .createReportingJob('printablePdfV2', decoratedJobParams) - .then(() => { + .then(() => firstValueFrom(startServices$)) + .then(([startServices]) => { toasts.addSuccess({ title: intl.formatMessage( { @@ -318,7 +313,7 @@ export const reportingExportModalProvider = ({ ), }} />, - { theme, i18n: i18nStart } + startServices ), 'data-test-subj': 'queueReportSuccess', }); @@ -347,7 +342,8 @@ export const reportingExportModalProvider = ({ }); return apiClient .createReportingJob('pngV2', decoratedJobParams) - .then(() => { + .then(() => firstValueFrom(startServices$)) + .then(([startServices]) => { toasts.addSuccess({ title: intl.formatMessage( { @@ -371,7 +367,7 @@ export const reportingExportModalProvider = ({ ), }} />, - { theme, i18n: i18nStart } + startServices ), 'data-test-subj': 'queueReportSuccess', }); @@ -414,7 +410,6 @@ export const reportingExportModalProvider = ({ /> ), layoutOption: objectType === 'dashboard' ? ('print' as const) : undefined, - theme, renderLayoutOptionSwitch: objectType === 'dashboard', renderCopyURLButton: true, absoluteUrl: new URL(relativePathPDF, window.location.href).toString(), diff --git a/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_reporting.tsx b/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_reporting.tsx index 6446efb787f665..15e671d2afc64b 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_reporting.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/register_pdf_png_reporting.tsx @@ -57,12 +57,10 @@ const getJobParams = export const reportingScreenshotShareProvider = ({ apiClient, - toasts, - uiSettings, license, application, usesUiCapabilities, - theme, + startServices$, }: ExportPanelShareOpts): ShareMenuProvider => { const getShareMenuItems = ({ objectType, @@ -150,15 +148,13 @@ export const reportingScreenshotShareProvider = ({ content: ( ), }, @@ -185,8 +181,6 @@ export const reportingScreenshotShareProvider = ({ content: ( ), }, diff --git a/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.test.tsx b/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.test.tsx index 67a7433fe08780..133ea782c0bdf2 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.test.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.test.tsx @@ -6,14 +6,10 @@ * Side Public License, v 1. */ -import { - httpServiceMock, - notificationServiceMock, - themeServiceMock, - uiSettingsServiceMock, -} from '@kbn/core/public/mocks'; +import { coreMock, httpServiceMock, uiSettingsServiceMock } from '@kbn/core/public/mocks'; import { mountWithIntl } from '@kbn/test-jest-helpers'; import React from 'react'; +import * as Rx from 'rxjs'; import { ReportingPanelProps as Props, ReportingPanelContent } from '.'; import { ReportingAPIClient } from '../../..'; import { ErrorUnsavedWorkPanel } from './components'; @@ -23,8 +19,6 @@ jest.mock('./constants', () => ({ getMaxUrlLength: jest.fn(() => 9999999), })); -const theme = themeServiceMock.createSetupContract(); - describe('ReportingPanelContent', () => { const props: Partial = { layoutId: 'super_cool_layout_id_X', @@ -34,7 +28,6 @@ describe('ReportingPanelContent', () => { objectType: 'noice_object', title: 'ultimate_title', }; - const toasts = notificationServiceMock.createSetupContract().toasts; const http = httpServiceMock.createSetupContract(); const uiSettings = uiSettingsServiceMock.createSetupContract(); let apiClient: ReportingAPIClient; @@ -50,6 +43,7 @@ describe('ReportingPanelContent', () => { apiClient = new ReportingAPIClient(http, uiSettings, '7.15.0-test'); }); + const { getStartServices } = coreMock.createSetup(); const mountComponent = (newProps: Partial) => mountWithIntl( { layoutId={props.layoutId} getJobParams={() => jobParams} apiClient={apiClient} - toasts={toasts} - uiSettings={uiSettings} - theme={theme} + startServices$={Rx.from(getStartServices())} {...props} {...newProps} /> diff --git a/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.tsx b/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.tsx index bafc355470fad2..a544e6a44464ba 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/reporting_panel_content/reporting_panel_content.tsx @@ -7,6 +7,7 @@ */ import React, { Component, ReactElement } from 'react'; +import * as Rx from 'rxjs'; import { CSV_REPORT_TYPE, CSV_REPORT_TYPE_V2 } from '@kbn/reporting-export-types-csv-common'; import { PDF_REPORT_TYPE, PDF_REPORT_TYPE_V2 } from '@kbn/reporting-export-types-pdf-common'; @@ -22,13 +23,14 @@ import { EuiSpacer, EuiText, } from '@elastic/eui'; -import { IUiSettingsClient, ThemeServiceSetup, ToastsSetup } from '@kbn/core/public'; + import { i18n } from '@kbn/i18n'; import { FormattedMessage, InjectedIntl, injectI18n } from '@kbn/i18n-react'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; import type { BaseParams } from '@kbn/reporting-common/types'; -import { ReportingAPIClient } from '../../../reporting_api_client'; +import type { StartServices } from '../..'; +import type { ReportingAPIClient } from '../../../reporting_api_client'; import { ErrorUnsavedWorkPanel, ErrorUrlTooLongPanel } from './components'; import { getMaxUrlLength } from './constants'; @@ -38,8 +40,6 @@ import { getMaxUrlLength } from './constants'; */ export interface ReportingPanelProps { apiClient: ReportingAPIClient; - toasts: ToastsSetup; - uiSettings: IUiSettingsClient; reportType: string; requiresSavedState: boolean; // Whether the report to be generated requires saved state that is not captured in the URL submitted to the report generator. @@ -51,7 +51,8 @@ export interface ReportingPanelProps { options?: ReactElement | null; isDirty?: boolean; onClose?: () => void; - theme: ThemeServiceSetup; + + startServices$: Rx.Observable; } export type Props = ReportingPanelProps & { intl: InjectedIntl }; @@ -277,68 +278,66 @@ class ReportingPanelContentUi extends Component { this.setState({ absoluteUrl }); }; - private createReportingJob = () => { - const { intl } = this.props; - const decoratedJobParams = this.props.apiClient.getDecoratedJobParams( - this.props.getJobParams() - ); + private createReportingJob = async () => { + const { startServices$, apiClient, intl } = this.props; + const [coreStart] = await Rx.firstValueFrom(startServices$); + const decoratedJobParams = apiClient.getDecoratedJobParams(this.props.getJobParams()); + const { toasts } = coreStart.notifications; this.setState({ isCreatingReportJob: true }); - return this.props.apiClient - .createReportingJob(this.props.reportType, decoratedJobParams) - .then(() => { - this.props.toasts.addSuccess({ - title: intl.formatMessage( - { - id: 'reporting.share.panelContent.successfullyQueuedReportNotificationTitle', - defaultMessage: 'Queued report for {objectType}', - }, - { objectType: this.state.objectType } - ), - text: toMountPoint( - - - - ), - }} - />, - { theme$: this.props.theme.theme$ } - ), - 'data-test-subj': 'queueReportSuccess', - }); - if (this.props.onClose) { - this.props.onClose(); - } - if (this.mounted) { - this.setState({ isCreatingReportJob: false }); - } - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.error(error); - this.props.toasts.addError(error, { - title: intl.formatMessage({ - id: 'reporting.share.panelContent.notification.reportingErrorTitle', - defaultMessage: 'Unable to create report', - }), - toastMessage: intl.formatMessage({ - id: 'reporting.share.panelContent.notification.reportingErrorToastMessage', - defaultMessage: `We couldn't create a report at this time.`, - }), - }); - if (this.mounted) { - this.setState({ isCreatingReportJob: false }); - } + try { + await this.props.apiClient.createReportingJob(this.props.reportType, decoratedJobParams); + toasts.addSuccess({ + title: intl.formatMessage( + { + id: 'reporting.share.panelContent.successfullyQueuedReportNotificationTitle', + defaultMessage: 'Queued report for {objectType}', + }, + { objectType: this.state.objectType } + ), + text: toMountPoint( + + + + ), + }} + />, + coreStart + ), + 'data-test-subj': 'queueReportSuccess', }); + if (this.props.onClose) { + this.props.onClose(); + } + if (this.mounted) { + this.setState({ isCreatingReportJob: false }); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + toasts.addError(error, { + title: intl.formatMessage({ + id: 'reporting.share.panelContent.notification.reportingErrorTitle', + defaultMessage: 'Unable to create report', + }), + toastMessage: intl.formatMessage({ + id: 'reporting.share.panelContent.notification.reportingErrorToastMessage', + defaultMessage: `We couldn't create a report at this time.`, + }), + }); + if (this.mounted) { + this.setState({ isCreatingReportJob: false }); + } + } }; } diff --git a/packages/kbn-reporting/public/share/share_context_menu/screen_capture_panel_content.test.tsx b/packages/kbn-reporting/public/share/share_context_menu/screen_capture_panel_content.test.tsx index 42d599da196226..854ac403e2d7c8 100644 --- a/packages/kbn-reporting/public/share/share_context_menu/screen_capture_panel_content.test.tsx +++ b/packages/kbn-reporting/public/share/share_context_menu/screen_capture_panel_content.test.tsx @@ -6,14 +6,16 @@ * Side Public License, v 1. */ -import { coreMock, themeServiceMock } from '@kbn/core/public/mocks'; +import * as Rx from 'rxjs'; +import { coreMock } from '@kbn/core/public/mocks'; import { __IntlProvider as IntlProvider } from '@kbn/i18n-react'; import { mount } from 'enzyme'; import React from 'react'; import { ReportingAPIClient } from '../..'; import { ScreenCapturePanelContent } from './screen_capture_panel_content'; -const { http, uiSettings, ...coreSetup } = coreMock.createSetup(); +const { http, uiSettings, getStartServices } = coreMock.createSetup(); +const startServices$ = Rx.from(getStartServices()); uiSettings.get.mockImplementation((key: string) => { switch (key) { case 'dateFormat:tz': @@ -28,8 +30,6 @@ const getJobParamsDefault = () => ({ browserTimezone: 'America/New_York', }); -const theme = themeServiceMock.createSetupContract(); - test('ScreenCapturePanelContent renders the default view properly', () => { const component = mount( @@ -37,10 +37,8 @@ test('ScreenCapturePanelContent renders the default view properly', () => { reportType="Analytical App" requiresSavedState={false} apiClient={apiClient} - uiSettings={uiSettings} - toasts={coreSetup.notifications.toasts} getJobParams={getJobParamsDefault} - theme={theme} + startServices$={startServices$} /> ); @@ -57,10 +55,8 @@ test('ScreenCapturePanelContent properly renders a view with "canvas" layout opt reportType="Analytical App" requiresSavedState={false} apiClient={apiClient} - uiSettings={uiSettings} - toasts={coreSetup.notifications.toasts} getJobParams={getJobParamsDefault} - theme={theme} + startServices$={startServices$} /> ); @@ -76,11 +72,9 @@ test('ScreenCapturePanelContent allows POST URL to be copied when objectId is pr reportType="Analytical App" requiresSavedState={false} apiClient={apiClient} - uiSettings={uiSettings} - toasts={coreSetup.notifications.toasts} getJobParams={getJobParamsDefault} objectId={'1234-5'} - theme={theme} + startServices$={startServices$} /> ); @@ -96,10 +90,8 @@ test('ScreenCapturePanelContent does not allow POST URL to be copied when object reportType="Analytical App" requiresSavedState={false} apiClient={apiClient} - uiSettings={uiSettings} - toasts={coreSetup.notifications.toasts} getJobParams={getJobParamsDefault} - theme={theme} + startServices$={startServices$} /> ); @@ -115,10 +107,8 @@ test('ScreenCapturePanelContent properly renders a view with "print" layout opti reportType="Analytical App" requiresSavedState={false} apiClient={apiClient} - uiSettings={uiSettings} - toasts={coreSetup.notifications.toasts} getJobParams={getJobParamsDefault} - theme={theme} + startServices$={startServices$} /> ); @@ -135,10 +125,8 @@ test('ScreenCapturePanelContent decorated job params are visible in the POST URL requiresSavedState={false} isDirty={false} apiClient={apiClient} - uiSettings={uiSettings} - toasts={coreSetup.notifications.toasts} getJobParams={getJobParamsDefault} - theme={theme} + startServices$={startServices$} /> ); diff --git a/packages/kbn-reporting/public/share/shared/get_shared_components.tsx b/packages/kbn-reporting/public/share/shared/get_shared_components.tsx index e9a4499071d97d..bc4ecc24281324 100644 --- a/packages/kbn-reporting/public/share/shared/get_shared_components.tsx +++ b/packages/kbn-reporting/public/share/shared/get_shared_components.tsx @@ -6,13 +6,17 @@ * Side Public License, v 1. */ -import { CoreSetup } from '@kbn/core/public'; -import { PDF_REPORT_TYPE_V2 } from '@kbn/reporting-export-types-pdf-common'; -import { PNG_REPORT_TYPE_V2 } from '@kbn/reporting-export-types-png-common'; import React from 'react'; +import { Observable } from 'rxjs'; + +import { PDF_REPORT_TYPE, PDF_REPORT_TYPE_V2 } from '@kbn/reporting-export-types-pdf-common'; +import { PNG_REPORT_TYPE, PNG_REPORT_TYPE_V2 } from '@kbn/reporting-export-types-png-common'; + +import { StartServices } from '..'; import { ReportingAPIClient } from '../..'; import { ReportingPanelProps } from '../share_context_menu/reporting_panel_content'; import { ScreenCapturePanelContent } from '../share_context_menu/screen_capture_panel_content_lazy'; + /** * Properties for displaying a share menu with Reporting features. */ @@ -53,24 +57,20 @@ export interface ReportingPublicComponents { * Related Discuss issue: https://github.com/elastic/kibana/issues/101422 */ export function getSharedComponents( - core: CoreSetup, - apiClient: ReportingAPIClient + apiClient: ReportingAPIClient, + startServices$: Observable ): ReportingPublicComponents { return { ReportingPanelPDFV2(props: ApplicationProps) { - const getJobParams = props.getJobParams as ReportingPanelProps['getJobParams']; if (props.layoutOption === 'canvas') { return ( ); } else { @@ -78,55 +78,43 @@ export function getSharedComponents( } }, ReportingPanelPNGV2(props: ApplicationProps) { - const getJobParams = props.getJobParams as ReportingPanelProps['getJobParams']; if (props.layoutOption === 'canvas') { return ( ); } }, ReportingModalPDF(props: ApplicationProps) { - const getJobParams = props.getJobParams as ReportingPanelProps['getJobParams']; if (props.layoutOption === 'canvas') { return ( ); } }, ReportingModalPNG(props: ApplicationProps) { - const getJobParams = props.getJobParams as ReportingPanelProps['getJobParams']; if (props.layoutOption === 'canvas') { return ( ); } diff --git a/packages/kbn-reporting/public/tsconfig.json b/packages/kbn-reporting/public/tsconfig.json index 7b36e7eeeb6163..1f17ff412c2869 100644 --- a/packages/kbn-reporting/public/tsconfig.json +++ b/packages/kbn-reporting/public/tsconfig.json @@ -31,5 +31,8 @@ "@kbn/i18n-react", "@kbn/test-jest-helpers", "@kbn/react-kibana-mount", + "@kbn/home-plugin", + "@kbn/management-plugin", + "@kbn/ui-actions-plugin", ] } diff --git a/packages/kbn-reporting/public/types.ts b/packages/kbn-reporting/public/types.ts index 67f5755e367cc1..82da5e5cfa0013 100644 --- a/packages/kbn-reporting/public/types.ts +++ b/packages/kbn-reporting/public/types.ts @@ -6,6 +6,22 @@ * Side Public License, v 1. */ +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { HomePublicPluginStart } from '@kbn/home-plugin/public'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/public'; +import type { ManagementStart } from '@kbn/management-plugin/public'; +import type { SharePluginStart } from '@kbn/share-plugin/public'; +import type { UiActionsStart } from '@kbn/ui-actions-plugin/public'; + +export interface ReportingPublicPluginStartDependencies { + home: HomePublicPluginStart; + data: DataPublicPluginStart; + management: ManagementStart; + licensing: LicensingPluginStart; + uiActions: UiActionsStart; + share: SharePluginStart; +} + export interface ClientConfigType { csv: { enablePanelActionDownload: boolean; diff --git a/x-pack/plugins/reporting/public/lib/stream_handler.test.ts b/x-pack/plugins/reporting/public/lib/stream_handler.test.ts index 37ef7967ae2876..88a624df552800 100644 --- a/x-pack/plugins/reporting/public/lib/stream_handler.test.ts +++ b/x-pack/plugins/reporting/public/lib/stream_handler.test.ts @@ -5,8 +5,7 @@ * 2.0. */ -import { NotificationsStart } from '@kbn/core/public'; -import { coreMock, docLinksServiceMock, themeServiceMock } from '@kbn/core/public/mocks'; +import { coreMock } from '@kbn/core/public/mocks'; import { JobId, ReportApiJSON } from '@kbn/reporting-common/types'; import { JobSummary, JobSummarySet } from '../types'; @@ -43,19 +42,10 @@ jobQueueClientMock.getError = () => Promise.resolve('this is the failed report e jobQueueClientMock.getManagementLink = () => '/#management'; jobQueueClientMock.getReportURL = () => '/reporting/download/job-123'; -const mockShowDanger = jest.fn(); -const mockShowSuccess = jest.fn(); -const mockShowWarning = jest.fn(); -const notificationsMock = { - toasts: { - addDanger: mockShowDanger, - addSuccess: mockShowSuccess, - addWarning: mockShowWarning, - }, -} as unknown as NotificationsStart; - -const theme = themeServiceMock.createStartContract(); -const docLink = docLinksServiceMock.createStartContract(); +const core = coreMock.createStart(); +const mockShowDanger = jest.spyOn(core.notifications.toasts, 'addDanger'); +const mockShowSuccess = jest.spyOn(core.notifications.toasts, 'addSuccess'); +const mockShowWarning = jest.spyOn(core.notifications.toasts, 'addWarning'); describe('stream handler', () => { afterEach(() => { @@ -63,23 +53,13 @@ describe('stream handler', () => { }); it('constructs', () => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); expect(sh).not.toBe(null); }); describe('findChangedStatusJobs', () => { it('finds no changed status jobs from empty', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); const findJobs = sh.testFindChangedStatusJobs([]); findJobs.subscribe((data) => { expect(data).toEqual({ completed: [], failed: [] }); @@ -88,12 +68,7 @@ describe('stream handler', () => { }); it('finds changed status jobs', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); const findJobs = sh.testFindChangedStatusJobs([ 'job-source-mock1', 'job-source-mock2', @@ -110,12 +85,7 @@ describe('stream handler', () => { describe('showNotifications', () => { it('show success', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); sh.testShowNotifications({ completed: [ { @@ -136,12 +106,7 @@ describe('stream handler', () => { }); it('show max length warning', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); sh.testShowNotifications({ completed: [ { @@ -163,12 +128,7 @@ describe('stream handler', () => { }); it('show csv formulas warning', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); sh.testShowNotifications({ completed: [ { @@ -190,12 +150,7 @@ describe('stream handler', () => { }); it('show failed job toast', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); sh.testShowNotifications({ completed: [], failed: [ @@ -216,12 +171,7 @@ describe('stream handler', () => { }); it('show multiple toast', (done) => { - const sh = new TestReportingNotifierStreamHandler( - notificationsMock, - jobQueueClientMock, - theme, - docLink - ); + const sh = new TestReportingNotifierStreamHandler(jobQueueClientMock, core); sh.testShowNotifications({ completed: [ { diff --git a/x-pack/plugins/reporting/public/lib/stream_handler.ts b/x-pack/plugins/reporting/public/lib/stream_handler.ts index 049aea96e1af2f..78513de46c8018 100644 --- a/x-pack/plugins/reporting/public/lib/stream_handler.ts +++ b/x-pack/plugins/reporting/public/lib/stream_handler.ts @@ -8,7 +8,7 @@ import * as Rx from 'rxjs'; import { catchError, filter, map, mergeMap, takeUntil } from 'rxjs'; -import { DocLinksStart, NotificationsSetup, ThemeServiceStart } from '@kbn/core/public'; +import { CoreStart } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import { JOB_STATUS } from '@kbn/reporting-common'; import { JobId } from '@kbn/reporting-common/types'; @@ -42,18 +42,14 @@ function getReportStatus(src: Job): JobSummary { }; } -function handleError( - err: Error, - notifications: NotificationsSetup, - theme: ThemeServiceStart -): Rx.Observable { - notifications.toasts.addDanger( +function handleError(core: CoreStart, err: Error): Rx.Observable { + core.notifications.toasts.addDanger( getGeneralErrorToast( i18n.translate('xpack.reporting.publicNotifier.pollingErrorMessage', { defaultMessage: 'Reporting notifier error!', }), err, - theme + core ) ); window.console.error(err); @@ -63,12 +59,7 @@ function handleError( export class ReportingNotifierStreamHandler { private jobCompletionNotifications = jobCompletionNotifications(); - constructor( - private notifications: NotificationsSetup, - private apiClient: ReportingAPIClient, - private theme: ThemeServiceStart, - private docLinks: DocLinksStart - ) {} + constructor(private apiClient: ReportingAPIClient, private core: CoreStart) {} public startPolling(interval: number, stop$: Rx.Observable) { Rx.timer(0, interval) @@ -81,7 +72,7 @@ export class ReportingNotifierStreamHandler { catchError((err) => { // eslint-disable-next-line no-console console.error(err); - return handleError(err, this.notifications, this.theme); + return handleError(this.core, err); }) ) .subscribe(); @@ -94,10 +85,10 @@ export class ReportingNotifierStreamHandler { completed: completedJobs, failed: failedJobs, }: JobSummarySet): Rx.Observable { - const notifications = this.notifications; + const notifications = this.core.notifications; const apiClient = this.apiClient; - const theme = this.theme; - const docLinks = this.docLinks; + const core = this.core; + const docLinks = this.core.docLinks; const getManagementLink = apiClient.getManagementLink.bind(apiClient); const getDownloadLink = apiClient.getDownloadLink.bind(apiClient); @@ -108,22 +99,22 @@ export class ReportingNotifierStreamHandler { for (const job of completedJobs ?? []) { if (job.csvContainsFormulas) { notifications.toasts.addWarning( - getWarningFormulasToast(job, getManagementLink, getDownloadLink, theme), + getWarningFormulasToast(job, getManagementLink, getDownloadLink, core), completedOptions ); } else if (job.maxSizeReached) { notifications.toasts.addWarning( - getWarningMaxSizeToast(job, getManagementLink, getDownloadLink, theme), + getWarningMaxSizeToast(job, getManagementLink, getDownloadLink, core), completedOptions ); } else if (job.status === JOB_STATUS.WARNINGS) { notifications.toasts.addWarning( - getWarningToast(job, getManagementLink, getDownloadLink, theme), + getWarningToast(job, getManagementLink, getDownloadLink, core), completedOptions ); } else { notifications.toasts.addSuccess( - getSuccessToast(job, getManagementLink, getDownloadLink, theme), + getSuccessToast(job, getManagementLink, getDownloadLink, core), completedOptions ); } @@ -132,8 +123,8 @@ export class ReportingNotifierStreamHandler { // no download link available for (const job of failedJobs ?? []) { const errorText = await apiClient.getError(job.id); - this.notifications.toasts.addDanger( - getFailureToast(errorText, job, getManagementLink, theme, docLinks) + notifications.toasts.addDanger( + getFailureToast(errorText, job, getManagementLink, docLinks, core) ); } return { completed: completedJobs, failed: failedJobs }; @@ -178,13 +169,13 @@ export class ReportingNotifierStreamHandler { }), catchError((err) => { // show connection refused toast - this.notifications.toasts.addDanger( + this.core.notifications.toasts.addDanger( getGeneralErrorToast( i18n.translate('xpack.reporting.publicNotifier.httpErrorMessage', { defaultMessage: 'Could not check Reporting job status!', }), err, - this.theme + this.core ) ); window.console.error(err); diff --git a/x-pack/plugins/reporting/public/management/mount_management_section.tsx b/x-pack/plugins/reporting/public/management/mount_management_section.tsx index 1e81119439e207..4352557e576173 100644 --- a/x-pack/plugins/reporting/public/management/mount_management_section.tsx +++ b/x-pack/plugins/reporting/public/management/mount_management_section.tsx @@ -10,11 +10,10 @@ import { render, unmountComponentAtNode } from 'react-dom'; import type { CoreStart } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { I18nProvider } from '@kbn/i18n-react'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/public'; import type { ManagementAppMountParams } from '@kbn/management-plugin/public'; -import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; import type { ClientConfigType } from '@kbn/reporting-public'; import type { SharePluginStart } from '@kbn/share-plugin/public'; import { @@ -44,25 +43,23 @@ export async function mountManagementSection( }; render( - - - - - - - - - - - , + + + + + + + + + , params.element ); diff --git a/x-pack/plugins/reporting/public/mocks.ts b/x-pack/plugins/reporting/public/mocks.ts index 34aa311e548227..b1a447f48a8c0e 100644 --- a/x-pack/plugins/reporting/public/mocks.ts +++ b/x-pack/plugins/reporting/public/mocks.ts @@ -5,6 +5,7 @@ * 2.0. */ +import * as Rx from 'rxjs'; import { coreMock } from '@kbn/core/public/mocks'; import { getSharedComponents } from '@kbn/reporting-public/share'; import { ReportingAPIClient } from '@kbn/reporting-public/reporting_api_client'; @@ -17,7 +18,7 @@ const createSetupContract = (): Setup => { const apiClient = new ReportingAPIClient(coreSetup.http, coreSetup.uiSettings, '7.15.0'); return { usesUiCapabilities: jest.fn().mockImplementation(() => true), - components: getSharedComponents(coreSetup, apiClient), + components: getSharedComponents(apiClient, Rx.from(coreSetup.getStartServices())), }; }; diff --git a/x-pack/plugins/reporting/public/notifier/general_error.tsx b/x-pack/plugins/reporting/public/notifier/general_error.tsx index a3a18edd454d7d..3764c5b6e84784 100644 --- a/x-pack/plugins/reporting/public/notifier/general_error.tsx +++ b/x-pack/plugins/reporting/public/notifier/general_error.tsx @@ -5,16 +5,16 @@ * 2.0. */ -import React from 'react'; -import { FormattedMessage } from '@kbn/i18n-react'; import { EuiCallOut, EuiSpacer } from '@elastic/eui'; -import { ThemeServiceStart, ToastInput } from '@kbn/core/public'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { CoreStart, ToastInput } from '@kbn/core/public'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { toMountPoint } from '@kbn/react-kibana-mount'; +import React from 'react'; export const getGeneralErrorToast = ( errorText: string, err: Error, - theme: ThemeServiceStart + core: CoreStart ): ToastInput => ({ text: toMountPoint( <> @@ -29,7 +29,7 @@ export const getGeneralErrorToast = ( defaultMessage="Try refreshing the page." /> , - { theme$: theme.theme$ } + core ), iconType: undefined, }); diff --git a/x-pack/plugins/reporting/public/notifier/job_failure.tsx b/x-pack/plugins/reporting/public/notifier/job_failure.tsx index e5c6f06413bdfd..c8f44931c29401 100644 --- a/x-pack/plugins/reporting/public/notifier/job_failure.tsx +++ b/x-pack/plugins/reporting/public/notifier/job_failure.tsx @@ -8,8 +8,8 @@ import { EuiCallOut, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import React from 'react'; -import { DocLinksStart, ThemeServiceStart, ToastInput } from '@kbn/core/public'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { CoreStart, DocLinksStart, ToastInput } from '@kbn/core/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; import * as errors from '@kbn/reporting-common/errors'; import { ManagementLinkFn } from '@kbn/reporting-common/types'; import { sharedI18nTexts } from '../shared_i18n_texts'; @@ -19,8 +19,8 @@ export const getFailureToast = ( errorText: string, job: JobSummary, getManagmenetLink: ManagementLinkFn, - theme: ThemeServiceStart, - docLinks: DocLinksStart + docLinks: DocLinksStart, + core: CoreStart ): ToastInput => { return { title: toMountPoint( @@ -29,7 +29,7 @@ export const getFailureToast = ( defaultMessage="Cannot create {reportType} report for '{reportObjectTitle}'." values={{ reportType: job.jobtype, reportObjectTitle: job.title }} />, - { theme$: theme.theme$ } + core ), text: toMountPoint( <> @@ -60,7 +60,7 @@ export const getFailureToast = ( />

, - { theme$: theme.theme$ } + core ), iconType: undefined, 'data-test-subj': 'completeReportFailure', diff --git a/x-pack/plugins/reporting/public/notifier/job_success.tsx b/x-pack/plugins/reporting/public/notifier/job_success.tsx index 00b08ed2413d90..ae721f675f605a 100644 --- a/x-pack/plugins/reporting/public/notifier/job_success.tsx +++ b/x-pack/plugins/reporting/public/notifier/job_success.tsx @@ -5,9 +5,9 @@ * 2.0. */ -import { ThemeServiceStart, ToastInput } from '@kbn/core/public'; +import { CoreStart, ToastInput } from '@kbn/core/public'; import { FormattedMessage } from '@kbn/i18n-react'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; import { JobId } from '@kbn/reporting-common/types'; import React from 'react'; import { JobSummary } from '../types'; @@ -18,7 +18,7 @@ export const getSuccessToast = ( job: JobSummary, getReportLink: () => string, getDownloadLink: (jobId: JobId) => string, - theme: ThemeServiceStart + core: CoreStart ): ToastInput => ({ title: toMountPoint( , - { theme$: theme.theme$ } + core ), color: 'success', text: toMountPoint( @@ -36,7 +36,7 @@ export const getSuccessToast = (

, - { theme$: theme.theme$ } + core ), 'data-test-subj': 'completeReportSuccess', }); diff --git a/x-pack/plugins/reporting/public/notifier/job_warning.tsx b/x-pack/plugins/reporting/public/notifier/job_warning.tsx index 6751eb76ab0734..34c73e561f976c 100644 --- a/x-pack/plugins/reporting/public/notifier/job_warning.tsx +++ b/x-pack/plugins/reporting/public/notifier/job_warning.tsx @@ -5,9 +5,9 @@ * 2.0. */ -import { ThemeServiceStart, ToastInput } from '@kbn/core/public'; +import { CoreStart, ToastInput } from '@kbn/core/public'; import { FormattedMessage } from '@kbn/i18n-react'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; import { JobId } from '@kbn/reporting-common/types'; import React from 'react'; import { JobSummary } from '../types'; @@ -18,7 +18,7 @@ export const getWarningToast = ( job: JobSummary, getReportLink: () => string, getDownloadLink: (jobId: JobId) => string, - theme: ThemeServiceStart + core: CoreStart ): ToastInput => ({ title: toMountPoint( , - { theme$: theme.theme$ } + core ), text: toMountPoint( <> @@ -35,7 +35,7 @@ export const getWarningToast = (

, - { theme$: theme.theme$ } + core ), 'data-test-subj': 'completeReportWarning', }); diff --git a/x-pack/plugins/reporting/public/notifier/job_warning_formulas.tsx b/x-pack/plugins/reporting/public/notifier/job_warning_formulas.tsx index 4cf9f3f655cc1f..2f8c39b9046663 100644 --- a/x-pack/plugins/reporting/public/notifier/job_warning_formulas.tsx +++ b/x-pack/plugins/reporting/public/notifier/job_warning_formulas.tsx @@ -7,9 +7,9 @@ import React from 'react'; -import { ThemeServiceStart, ToastInput } from '@kbn/core/public'; +import { CoreStart, ToastInput } from '@kbn/core/public'; import { FormattedMessage } from '@kbn/i18n-react'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; import { JobId } from '@kbn/reporting-common/types'; import { DownloadButton } from './job_download_button'; @@ -20,7 +20,7 @@ export const getWarningFormulasToast = ( job: JobSummary, getReportLink: () => string, getDownloadLink: (jobId: JobId) => string, - theme: ThemeServiceStart + core: CoreStart ): ToastInput => ({ title: toMountPoint( , - { theme$: theme.theme$ } + core ), text: toMountPoint( <> @@ -44,7 +44,7 @@ export const getWarningFormulasToast = (

, - { theme$: theme.theme$ } + core ), 'data-test-subj': 'completeReportCsvFormulasWarning', }); diff --git a/x-pack/plugins/reporting/public/notifier/job_warning_max_size.tsx b/x-pack/plugins/reporting/public/notifier/job_warning_max_size.tsx index 54c76282420676..b87547669d704a 100644 --- a/x-pack/plugins/reporting/public/notifier/job_warning_max_size.tsx +++ b/x-pack/plugins/reporting/public/notifier/job_warning_max_size.tsx @@ -5,9 +5,9 @@ * 2.0. */ -import { ThemeServiceStart, ToastInput } from '@kbn/core/public'; +import { CoreStart, ToastInput } from '@kbn/core/public'; import { FormattedMessage } from '@kbn/i18n-react'; -import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { toMountPoint } from '@kbn/react-kibana-mount'; import type { JobId } from '@kbn/reporting-common/types'; import React from 'react'; import { JobSummary } from '../types'; @@ -18,7 +18,7 @@ export const getWarningMaxSizeToast = ( job: JobSummary, getReportLink: () => string, getDownloadLink: (jobId: JobId) => string, - theme: ThemeServiceStart + core: CoreStart ): ToastInput => ({ title: toMountPoint( , - { theme$: theme.theme$ } + core ), text: toMountPoint( <> @@ -41,7 +41,7 @@ export const getWarningMaxSizeToast = (

, - { theme$: theme.theme$ } + core ), 'data-test-subj': 'completeReportMaxSizeWarning', }); diff --git a/x-pack/plugins/reporting/public/plugin.ts b/x-pack/plugins/reporting/public/plugin.ts index dddf18003fb946..cb606ca35152f5 100644 --- a/x-pack/plugins/reporting/public/plugin.ts +++ b/x-pack/plugins/reporting/public/plugin.ts @@ -5,16 +5,9 @@ * 2.0. */ -import { from, ReplaySubject } from 'rxjs'; +import { from, map, type Observable, ReplaySubject } from 'rxjs'; -import { - CoreSetup, - CoreStart, - HttpSetup, - IUiSettingsClient, - Plugin, - PluginInitializerContext, -} from '@kbn/core/public'; +import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { CONTEXT_MENU_TRIGGER } from '@kbn/embeddable-plugin/public'; import type { HomePublicPluginSetup, HomePublicPluginStart } from '@kbn/home-plugin/public'; @@ -39,6 +32,7 @@ import { import { ReportingCsvPanelAction } from '@kbn/reporting-csv-share-panel'; import type { ReportingSetup, ReportingStart } from '.'; import { ReportingNotifierStreamHandler as StreamHandler } from './lib/stream_handler'; +import { StartServices } from './types'; export interface ReportingPublicPluginSetupDependencies { home: HomePublicPluginSetup; @@ -57,6 +51,8 @@ export interface ReportingPublicPluginStartDependencies { share: SharePluginStart; } +type StartServices$ = Observable; + /** * @internal * @implements Plugin @@ -81,29 +77,18 @@ export class ReportingPublicPlugin }); private config: ClientConfigType; private contract?: ReportingSetup; + private startServices$?: StartServices$; constructor(initializerContext: PluginInitializerContext) { this.config = initializerContext.config.get(); this.kibanaVersion = initializerContext.env.packageInfo.version; } - /* - * Use a single instance of ReportingAPIClient for all the reporting code - */ - private getApiClient(http: HttpSetup, uiSettings: IUiSettingsClient) { - if (!this.apiClient) { - this.apiClient = new ReportingAPIClient(http, uiSettings, this.kibanaVersion); - } - return this.apiClient; - } - - private getContract(core?: CoreSetup) { - if (core) { - this.contract = { - usesUiCapabilities: () => this.config.roles?.enabled === false, - components: getSharedComponents(core, this.getApiClient(core.http, core.uiSettings)), - }; - } + private getContract(apiClient: ReportingAPIClient, startServices$: StartServices$) { + this.contract = { + usesUiCapabilities: () => this.config.roles?.enabled === false, + components: getSharedComponents(apiClient, startServices$), + }; if (!this.contract) { throw new Error(`Setup error in Reporting plugin!`); @@ -116,7 +101,7 @@ export class ReportingPublicPlugin core: CoreSetup, setupDeps: ReportingPublicPluginSetupDependencies ) { - const { getStartServices, uiSettings } = core; + const { getStartServices } = core; const { home: homeSetup, management: managementSetup, @@ -125,10 +110,25 @@ export class ReportingPublicPlugin uiActions: uiActionsSetup, } = setupDeps; - const startServices$ = from(getStartServices()); + const startServices$: Observable = from(getStartServices()).pipe( + map(([services, ...rest]) => { + return [ + { + application: services.application, + analytics: services.analytics, + i18n: services.i18n, + theme: services.theme, + notifications: services.notifications, + uiSettings: services.uiSettings, + }, + ...rest, + ]; + }) + ); const usesUiCapabilities = !this.config.roles.enabled; - const apiClient = this.getApiClient(core.http, core.uiSettings); + const apiClient = new ReportingAPIClient(core.http, core.uiSettings, this.kibanaVersion); + this.apiClient = apiClient; homeSetup.featureCatalogue.register({ id: 'reporting', @@ -204,20 +204,15 @@ export class ReportingPublicPlugin }) ); - const reportingStart = this.getContract(core); - const { toasts } = core.notifications; - - startServices$.subscribe(([{ application, i18n: i18nStart }, { licensing }]) => { + startServices$.subscribe(([{ application }, { licensing }]) => { licensing.license$.subscribe((license) => { shareSetup.register( reportingCsvShareProvider({ apiClient, - toasts, - uiSettings, license, application, usesUiCapabilities, - theme: core.theme, + startServices$, }) ); if (this.config.export_types.pdf.enabled || this.config.export_types.png.enabled) { @@ -225,12 +220,10 @@ export class ReportingPublicPlugin shareSetup.register( reportingScreenshotShareProvider({ apiClient, - toasts, - uiSettings, license, application, usesUiCapabilities, - theme: core.theme, + startServices$, }) ); } @@ -238,12 +231,10 @@ export class ReportingPublicPlugin shareSetup.register( reportingCsvShareModalProvider({ apiClient, - uiSettings, license, application, usesUiCapabilities, - theme: core.theme, - i18n: i18nStart, + startServices$, }) ); @@ -251,29 +242,27 @@ export class ReportingPublicPlugin shareSetup.register( reportingExportModalProvider({ apiClient, - uiSettings, license, application, usesUiCapabilities, - theme: core.theme, - i18n: i18nStart, + startServices$, }) ); } } }); }); - return reportingStart; + + this.startServices$ = startServices$; + return this.getContract(apiClient, startServices$); } public start(core: CoreStart) { - const { notifications, docLinks } = core; - const apiClient = this.getApiClient(core.http, core.uiSettings); - const streamHandler = new StreamHandler(notifications, apiClient, core.theme, docLinks); + const streamHandler = new StreamHandler(this.apiClient!, core); const interval = durationToNumber(this.config.poll.jobsRefresh.interval); streamHandler.startPolling(interval, this.stop$); - return this.getContract(); + return this.getContract(this.apiClient!, this.startServices$!); } public stop() { diff --git a/x-pack/plugins/reporting/public/types.ts b/x-pack/plugins/reporting/public/types.ts index d5af032db617ce..9ba50435471ab2 100644 --- a/x-pack/plugins/reporting/public/types.ts +++ b/x-pack/plugins/reporting/public/types.ts @@ -5,8 +5,29 @@ * 2.0. */ +import type { CoreStart } from '@kbn/core/public'; import { JOB_STATUS } from '@kbn/reporting-common'; import type { JobId, ReportOutput, ReportSource, TaskRunResult } from '@kbn/reporting-common/types'; +import { ReportingPublicPluginStartDependencies } from './plugin'; + +/* + * Required services for mounting React components + */ +export type StartServices = [ + Pick< + CoreStart, + // required for modules that render React + | 'analytics' + | 'i18n' + | 'theme' + // used extensively in Reporting plugin + | 'application' + | 'notifications' + | 'uiSettings' + >, + ReportingPublicPluginStartDependencies, + unknown +]; /* * Notifier Toasts diff --git a/x-pack/plugins/reporting/tsconfig.json b/x-pack/plugins/reporting/tsconfig.json index 867233ad463b0c..e0f781d28f62cc 100644 --- a/x-pack/plugins/reporting/tsconfig.json +++ b/x-pack/plugins/reporting/tsconfig.json @@ -43,13 +43,14 @@ "@kbn/reporting-export-types-png", "@kbn/reporting-export-types-pdf-common", "@kbn/reporting-export-types-csv-common", - "@kbn/react-kibana-context-theme", "@kbn/reporting-export-types-png-common", "@kbn/reporting-mocks-server", "@kbn/core-http-request-handler-context-server", "@kbn/reporting-public", "@kbn/analytics-client", "@kbn/reporting-csv-share-panel", + "@kbn/react-kibana-context-render", + "@kbn/react-kibana-mount", ], "exclude": [ "target/**/*", From d13d89ecb8b948e9a8ca6bba193c5af5ec587cfc Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Thu, 25 Apr 2024 12:06:16 -0400 Subject: [PATCH 112/138] [Investigations] - Add unified components to eql tab (#180972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR introduces the unified table and field list to the correlations tab as seen in the snapshots below. **As of this PR, items that are not working** 1. Table row height controls 2. Expandable flyout integration 3. Leading cell actions (pinning, notes, row actions, analyzer, session view) **Changes in this PR:** Sequence Highlighting: image Building block highlighting: Screenshot 2024-04-17 at 1 11 13 PM To test: 1. Add `xpack.securitySolution.enableExperimental: [unifiedComponentsInTimelineEnabled]` to your `kibana.dev.yml` 2. Generate test data (endpoint data is fine) 5. Go to the correlations tab and enter this query to see default events/alerts that should have no highlighting ```any where true``` 6. Enter this query to see a generic sequence ``` sequence [any where true] [any where true] ``` You can also do something like ``` sequence [any where host.name=={HOST NAME VALUE HERE}] [any where true] ``` 7. You can also create a correlation rule using any of the above queries to generate building block alerts, and then query those alerts in the correlations tab as well --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../timeline/body/unified_timeline_body.tsx | 2 + .../timeline/tabs/eql/header/index.test.tsx | 64 +++ .../timeline/tabs/eql/header/index.tsx | 68 +++ .../components/timeline/tabs/eql/index.tsx | 327 ++++++------ .../timeline/tabs/query/header/index.test.tsx | 5 +- .../timeline/tabs/query/header/index.tsx | 111 ++-- .../components/timeline/tabs/query/index.tsx | 158 ++---- .../use_timeline_columns.test.ts.snap | 497 ++++++++++++++++++ .../tabs/shared/use_timeline_columns.test.ts | 152 ++++++ .../tabs/shared/use_timeline_columns.ts | 79 +++ .../components/timeline/tabs/shared/utils.ts | 18 +- ...stom_timeline_data_grid_body.test.tsx.snap | 4 +- .../custom_timeline_data_grid_body.tsx | 16 +- .../unified_components/data_table/index.tsx | 18 +- .../use_get_event_type_row_classname.test.ts | 57 ++ .../use_get_event_type_row_classname.ts | 34 ++ .../timeline/unified_components/index.tsx | 3 + .../timeline/unified_components/styles.tsx | 20 +- 18 files changed, 1291 insertions(+), 342 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/header/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/header/index.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/shared/__snapshots__/use_timeline_columns.test.ts.snap create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/shared/use_timeline_columns.test.ts create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/shared/use_timeline_columns.ts create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/use_get_event_type_row_classname.test.ts create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/use_get_event_type_row_classname.ts diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/unified_timeline_body.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/unified_timeline_body.tsx index 1a131871dc4fe7..21713d10b61f15 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/unified_timeline_body.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/unified_timeline_body.tsx @@ -22,6 +22,7 @@ export interface UnifiedTimelineBodyProps extends ComponentProps { const { header, + isSortEnabled, pageInfo, columns, rowRenderers, @@ -68,6 +69,7 @@ export const UnifiedTimelineBody = (props: UnifiedTimelineBodyProps) => { { + const props = { + activeTab: TimelineTabs.eql, + timelineId: TimelineId.test, + timelineFullScreen: false, + setTimelineFullScreen: jest.fn(), + } as EqlTabHeaderProps; + + describe('rendering', () => { + beforeEach(() => { + render( + + + + ); + }); + + test('should render the eql query bar', async () => { + expect(screen.getByTestId('EqlQueryBarTimeline')).toBeInTheDocument(); + }); + + test('should render the sourcerer selector', async () => { + expect(screen.getByTestId('timeline-sourcerer-popover')).toBeInTheDocument(); + }); + + test('should render the date picker', async () => { + expect(screen.getByTestId('superDatePickerToggleQuickMenuButton')).toBeInTheDocument(); + }); + }); + + describe('full screen', () => { + beforeEach(() => { + const updatedProps = { + ...props, + timelineFullScreen: true, + } as EqlTabHeaderProps; + + render( + + + + ); + }); + + test('should render the exit full screen component', async () => { + expect(screen.getByTestId('exit-full-screen')).toBeInTheDocument(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/header/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/header/index.tsx new file mode 100644 index 00000000000000..4ba340cb9f5cdc --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/header/index.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import React, { memo } from 'react'; + +import { InputsModelId } from '../../../../../../common/store/inputs/constants'; +import { TimelineTabs } from '../../../../../../../common/types/timeline'; +import { ExitFullScreen } from '../../../../../../common/components/exit_full_screen'; +import { SuperDatePicker } from '../../../../../../common/components/super_date_picker'; +import { SourcererScopeName } from '../../../../../../common/store/sourcerer/model'; +import { TimelineDatePickerLock } from '../../../date_picker_lock'; +import type { TimelineFullScreen } from '../../../../../../common/containers/use_full_screen'; +import { EqlQueryBarTimeline } from '../../../query_bar/eql'; +import { Sourcerer } from '../../../../../../common/components/sourcerer'; +import { StyledEuiFlyoutHeader, TabHeaderContainer } from '../../shared/layout'; + +export type EqlTabHeaderProps = { + activeTab: TimelineTabs; + timelineId: string; +} & TimelineFullScreen; + +export const EqlTabHeader = memo( + ({ activeTab, setTimelineFullScreen, timelineFullScreen, timelineId }: EqlTabHeaderProps) => ( + <> + + + + + {timelineFullScreen && setTimelineFullScreen != null && ( + + )} + + {activeTab === TimelineTabs.eql && ( + + )} + + + + + + + + + + + + + + + + ) +); + +EqlTabHeader.displayName = 'EqlTabHeader'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx index d11fa0e84b7d70..83a7487212e61c 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx @@ -15,20 +15,17 @@ import deepEqual from 'fast-deep-equal'; import { InPortal } from 'react-reverse-portal'; import { DataLoadingState } from '@kbn/unified-data-table'; -import type { ControlColumnProps } from '../../../../../../common/types'; import { InputsModelId } from '../../../../../common/store/inputs/constants'; +import { useDeepEqualSelector } from '../../../../../common/hooks/use_selector'; +import { useIsExperimentalFeatureEnabled } from '../../../../../common/hooks/use_experimental_features'; import { timelineActions, timelineSelectors } from '../../../../store'; import { useTimelineEvents } from '../../../../containers'; -import { defaultHeaders } from '../../body/column_headers/default_headers'; import { StatefulBody } from '../../body'; import { Footer, footerHeight } from '../../footer'; import { calculateTotalPages } from '../../helpers'; import { TimelineRefetch } from '../../refetch_timeline'; import type { ToggleDetailPanel } from '../../../../../../common/types/timeline'; -import { TimelineTabs } from '../../../../../../common/types/timeline'; -import { requiredFieldsForActions } from '../../../../../detections/components/alerts_table/default_config'; -import { ExitFullScreen } from '../../../../../common/components/exit_full_screen'; -import { SuperDatePicker } from '../../../../../common/components/super_date_picker'; +import { TimelineId, TimelineTabs } from '../../../../../../common/types/timeline'; import { EventDetailsWidthProvider } from '../../../../../common/components/events_viewer/event_details_width_context'; import type { inputsModel, State } from '../../../../../common/store'; import { inputsSelectors } from '../../../../../common/store'; @@ -37,34 +34,29 @@ import { timelineDefaults } from '../../../../store/defaults'; import { useSourcererDataView } from '../../../../../common/containers/sourcerer'; import { useEqlEventsCountPortal } from '../../../../../common/hooks/use_timeline_events_count'; import type { TimelineModel } from '../../../../store/model'; -import { TimelineDatePickerLock } from '../../date_picker_lock'; import { useTimelineFullScreen } from '../../../../../common/containers/use_full_screen'; import { DetailsPanel } from '../../../side_panel'; -import { EqlQueryBarTimeline } from '../../query_bar/eql'; -import { getDefaultControlColumn } from '../../body/control_columns'; -import type { Sort } from '../../body/sort'; -import { Sourcerer } from '../../../../../common/components/sourcerer'; -import { useLicense } from '../../../../../common/hooks/use_license'; -import { HeaderActions } from '../../../../../common/components/header_actions/header_actions'; import { EventsCountBadge, FullWidthFlexGroup, ScrollableFlexItem, - StyledEuiFlyoutHeader, StyledEuiFlyoutBody, StyledEuiFlyoutFooter, VerticalRule, - TabHeaderContainer, } from '../shared/layout'; -import { EMPTY_EVENTS, isTimerangeSame } from '../shared/utils'; +import { + TIMELINE_EMPTY_EVENTS, + isTimerangeSame, + timelineEmptyTrailingControlColumns, + TIMELINE_NO_SORTING, +} from '../shared/utils'; import type { TimelineTabCommonProps } from '../shared/types'; +import { UnifiedTimelineBody } from '../../body/unified_timeline_body'; +import { EqlTabHeader } from './header'; +import { useTimelineColumns } from '../shared/use_timeline_columns'; export type Props = TimelineTabCommonProps & PropsFromRedux; -const NO_SORTING: Sort[] = []; - -const trailingControlColumns: ControlColumnProps[] = []; // stable reference - export const EqlTabContentComponent: React.FC = ({ activeTab, columns, @@ -93,39 +85,46 @@ export const EqlTabContentComponent: React.FC = ({ runtimeMappings, selectedPatterns, } = useSourcererDataView(SourcererScopeName.timeline); + const { augmentedColumnHeaders, getTimelineQueryFieldsFromColumns, leadingControlColumns } = + useTimelineColumns(columns); - const isEnterprisePlus = useLicense().isEnterprise(); - const ACTION_BUTTON_COUNT = isEnterprisePlus ? 6 : 5; + const unifiedComponentsInTimelineEnabled = useIsExperimentalFeatureEnabled( + 'unifiedComponentsInTimelineEnabled' + ); - const isBlankTimeline: boolean = isEmpty(eqlQuery); + const getManageTimeline = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); - const canQueryTimeline = () => - loadingSourcerer != null && - !loadingSourcerer && - !isEmpty(start) && - !isEmpty(end) && - !isBlankTimeline; + const currentTimeline = useDeepEqualSelector((state) => + getManageTimeline(state, timelineId ?? TimelineId.active) + ); - const getTimelineQueryFields = () => { - const columnsHeader = isEmpty(columns) ? defaultHeaders : columns; - const columnFields = columnsHeader.map((c) => c.id); + const { sampleSize } = currentTimeline; - return [...columnFields, ...requiredFieldsForActions]; - }; + const isBlankTimeline: boolean = isEmpty(eqlQuery); + + const canQueryTimeline = useCallback( + () => + loadingSourcerer != null && + !loadingSourcerer && + !isEmpty(start) && + !isEmpty(end) && + !isBlankTimeline, + [end, isBlankTimeline, loadingSourcerer, start] + ); const [ - queryLoadingState, + dataLoadingState, { events, inspect, totalCount, pageInfo, loadPage, refreshedAt, refetch }, ] = useTimelineEvents({ dataViewId, endDate: end, eqlOptions: restEqlOption, - fields: getTimelineQueryFields(), + fields: getTimelineQueryFieldsFromColumns(), filterQuery: eqlQuery ?? '', id: timelineId, indexNames: selectedPatterns, language: 'eql', - limit: itemsPerPage, + limit: unifiedComponentsInTimelineEnabled ? sampleSize : itemsPerPage, runtimeMappings, skip: !canQueryTimeline(), startDate: start, @@ -134,9 +133,9 @@ export const EqlTabContentComponent: React.FC = ({ const isQueryLoading = useMemo( () => - queryLoadingState === DataLoadingState.loading || - queryLoadingState === DataLoadingState.loadingMore, - [queryLoadingState] + dataLoadingState === DataLoadingState.loading || + dataLoadingState === DataLoadingState.loadingMore, + [dataLoadingState] ); const handleOnPanelClosed = useCallback(() => { @@ -152,136 +151,142 @@ export const EqlTabContentComponent: React.FC = ({ ); }, [loadingSourcerer, timelineId, isQueryLoading, dispatch]); - const leadingControlColumns = useMemo( - () => - getDefaultControlColumn(ACTION_BUTTON_COUNT).map((x) => ({ - ...x, - headerCellRender: HeaderActions, - })), - [ACTION_BUTTON_COUNT] + const unifiedHeader = useMemo( + () => ( + + + + ), + [activeTab, setTimelineFullScreen, timelineFullScreen, timelineId] ); return ( <> - - {totalCount >= 0 ? {totalCount} : null} - - - - - - - - - - {timelineFullScreen && setTimelineFullScreen != null && ( - + + {totalCount >= 0 ? {totalCount} : null} + + + + + + + + ) : ( + <> + + {totalCount >= 0 ? {totalCount} : null} + + + + + + + + + + - )} - - {activeTab === TimelineTabs.eql && ( - + + + + {!isBlankTimeline && ( +