From 563c7f119361dd2fe8d8adead2cea9a257f8e2c6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 5 Nov 2024 22:58:40 +1100 Subject: [PATCH] [8.16] [Security GenAI][BUG] Knowledge Base: Show only indices with `semantic_text` fields (#198707) (#198906) # Backport This will backport the following commits from `main` to `8.16`: - [[Security GenAI][BUG] Knowledge Base: Show only indices with `semantic_text` fields (#198707)](https://github.com/elastic/kibana/pull/198707) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Ievgen Sorokopud --- .../kbn-elastic-assistant-common/constants.ts | 2 + .../impl/schemas/index.ts | 1 + .../get_knowledge_base_indices_route.gen.ts | 25 +++ ...t_knowledge_base_indices_route.schema.yaml | 42 +++++ .../assistant/api/knowledge_base/api.test.tsx | 32 +++- .../impl/assistant/api/knowledge_base/api.tsx | 31 ++++ .../use_knowledge_base_indices.test.tsx | 85 +++++++++ .../use_knowledge_base_indices.tsx | 59 +++++++ .../index.test.tsx | 6 +- .../index.tsx | 1 + .../index_entry_editor.test.tsx | 13 +- .../index_entry_editor.tsx | 28 +-- .../translations.ts | 8 + .../server/__mocks__/request.ts | 7 + .../elastic_assistant/server/routes/index.ts | 1 + .../get_knowledge_base_indices.test.ts | 79 +++++++++ .../get_knowledge_base_indices.ts | 73 ++++++++ .../server/routes/register_routes.ts | 2 + .../semantic_text_fields/data.json | 161 ++++++++++++++++++ .../semantic_text_fields/mappings.json | 30 ++++ .../trial_license_complete_tier/index.ts | 1 + .../semntic_text_indices.ts | 38 +++++ .../knowledge_base/entries/utils/helpers.ts | 30 ++++ 23 files changed, 738 insertions(+), 17 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.gen.ts create mode 100644 x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.schema.yaml create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.test.tsx create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.tsx create mode 100644 x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.ts create mode 100644 x-pack/test/functional/es_archives/security_solution/semantic_text_fields/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/semantic_text_fields/mappings.json create mode 100644 x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/semntic_text_indices.ts diff --git a/x-pack/packages/kbn-elastic-assistant-common/constants.ts b/x-pack/packages/kbn-elastic-assistant-common/constants.ts index d84d9d4cd6825..49db6c295a51a 100755 --- a/x-pack/packages/kbn-elastic-assistant-common/constants.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/constants.ts @@ -50,6 +50,8 @@ export const ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_ENTRIES_URL_FIND = `${ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_ENTRIES_URL}/_find` as const; export const ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_ENTRIES_URL_BULK_ACTION = `${ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_ENTRIES_URL}/_bulk_action` as const; +export const ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL = + `${ELASTIC_AI_ASSISTANT_INTERNAL_URL}/knowledge_base/_indices` as const; export const ELASTIC_AI_ASSISTANT_EVALUATE_URL = `${ELASTIC_AI_ASSISTANT_INTERNAL_URL}/evaluate` as const; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts index 6304bfa4786cf..9233791a870c3 100644 --- a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts @@ -49,6 +49,7 @@ export * from './actions_connector/post_actions_connector_execute_route.gen'; // Knowledge Base Schemas export * from './knowledge_base/crud_kb_route.gen'; +export * from './knowledge_base/get_knowledge_base_indices_route.gen'; export * from './knowledge_base/entries/bulk_crud_knowledge_base_entries_route.gen'; export * from './knowledge_base/entries/common_attributes.gen'; export * from './knowledge_base/entries/crud_knowledge_base_entries_route.gen'; diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.gen.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.gen.ts new file mode 100644 index 0000000000000..0e1df8bce089f --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.gen.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Get Knowledge Base Indices API endpoints + * version: 1 + */ + +import { z } from '@kbn/zod'; + +export type GetKnowledgeBaseIndicesResponse = z.infer; +export const GetKnowledgeBaseIndicesResponse = z.object({ + /** + * List of indices with at least one field of a `sematic_text` type. + */ + indices: z.array(z.string()), +}); diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.schema.yaml b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.schema.yaml new file mode 100644 index 0000000000000..f9dba830f9556 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/get_knowledge_base_indices_route.schema.yaml @@ -0,0 +1,42 @@ +openapi: 3.0.0 +info: + title: Get Knowledge Base Indices API endpoints + version: '1' +paths: + /internal/elastic_assistant/knowledge_base/_indices: + get: + x-codegen-enabled: true + x-labels: [ess, serverless] + operationId: GetKnowledgeBaseIndices + description: Gets Knowledge Base indices that have fields of a `sematic_text` type. + summary: Gets Knowledge Base indices that have fields of a `sematic_text` type. + tags: + - KnowledgeBase API + responses: + 200: + description: Indicates a successful call. + content: + application/json: + schema: + type: object + properties: + indices: + type: array + description: List of indices with at least one field of a `sematic_text` type. + items: + type: string + required: + - indices + 400: + description: Generic Error + content: + application/json: + schema: + type: object + properties: + statusCode: + type: number + error: + type: string + message: + type: string diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx index 22ccd2bc0ecdf..5509f43037444 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.test.tsx @@ -7,7 +7,12 @@ import { HttpSetup } from '@kbn/core-http-browser'; -import { deleteKnowledgeBase, getKnowledgeBaseStatus, postKnowledgeBase } from './api'; +import { + deleteKnowledgeBase, + getKnowledgeBaseIndices, + getKnowledgeBaseStatus, + postKnowledgeBase, +} from './api'; jest.mock('@kbn/core-http-browser'); @@ -95,4 +100,29 @@ describe('API tests', () => { await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error'); }); }); + + describe('getKnowledgeBaseIndices', () => { + it('calls the knowledge base API when correct resource path', async () => { + await getKnowledgeBaseIndices({ http: mockHttp }); + + expect(mockHttp.fetch).toHaveBeenCalledWith( + '/internal/elastic_assistant/knowledge_base/_indices', + { + method: 'GET', + signal: undefined, + version: '1', + } + ); + }); + it('returns error when error is an error', async () => { + const error = 'simulated error'; + (mockHttp.fetch as jest.Mock).mockImplementation(() => { + throw new Error(error); + }); + + await expect(getKnowledgeBaseIndices({ http: mockHttp })).resolves.toThrowError( + 'simulated error' + ); + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx index 4dd03a1cb2931..4db8c0787a1e1 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/api.tsx @@ -11,7 +11,9 @@ import { CreateKnowledgeBaseResponse, DeleteKnowledgeBaseRequestParams, DeleteKnowledgeBaseResponse, + ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, + GetKnowledgeBaseIndicesResponse, ReadKnowledgeBaseRequestParams, ReadKnowledgeBaseResponse, } from '@kbn/elastic-assistant-common'; @@ -108,3 +110,32 @@ export const deleteKnowledgeBase = async ({ return error as IHttpFetchError; } }; + +/** + * API call for getting indices that have fields of `semantic_text` type. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {AbortSignal} [options.signal] - AbortSignal + * + * @returns {Promise} + */ +export const getKnowledgeBaseIndices = async ({ + http, + signal, +}: { + http: HttpSetup; + signal?: AbortSignal | undefined; +}): Promise => { + try { + const response = await http.fetch(ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, { + method: 'GET', + signal, + version: API_VERSIONS.internal.v1, + }); + + return response as GetKnowledgeBaseIndicesResponse; + } catch (error) { + return error as IHttpFetchError; + } +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.test.tsx new file mode 100644 index 0000000000000..4f258aa3c1964 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.test.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act, renderHook } from '@testing-library/react-hooks'; +import { + useKnowledgeBaseIndices, + UseKnowledgeBaseIndicesParams, +} from './use_knowledge_base_indices'; +import { getKnowledgeBaseIndices as _getKnowledgeBaseIndices } from './api'; + +const getKnowledgeBaseIndicesMock = _getKnowledgeBaseIndices as jest.Mock; + +jest.mock('./api', () => { + const actual = jest.requireActual('./api'); + return { + ...actual, + getKnowledgeBaseIndices: jest.fn((...args) => actual.getKnowledgeBaseIndices(...args)), + }; +}); + +jest.mock('@tanstack/react-query', () => ({ + useQuery: jest.fn().mockImplementation(async (queryKey, fn, opts) => { + try { + const res = await fn({}); + return Promise.resolve(res); + } catch (e) { + opts.onError(e); + } + }), +})); + +const indicesResponse = ['index-1', 'index-2', 'index-3']; + +const http = { + fetch: jest.fn().mockResolvedValue(indicesResponse), +}; +const toasts = { + addError: jest.fn(), +}; +const defaultProps = { http, toasts } as unknown as UseKnowledgeBaseIndicesParams; +describe('useKnowledgeBaseIndices', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should call api to get knowledge base indices', async () => { + await act(async () => { + const { waitForNextUpdate } = renderHook(() => useKnowledgeBaseIndices(defaultProps)); + await waitForNextUpdate(); + + expect(defaultProps.http.fetch).toHaveBeenCalledWith( + '/internal/elastic_assistant/knowledge_base/_indices', + { + method: 'GET', + signal: undefined, + version: '1', + } + ); + expect(toasts.addError).not.toHaveBeenCalled(); + }); + }); + + it('should return indices response', async () => { + await act(async () => { + const { result, waitForNextUpdate } = renderHook(() => useKnowledgeBaseIndices(defaultProps)); + await waitForNextUpdate(); + + await expect(result.current).resolves.toStrictEqual(indicesResponse); + }); + }); + + it('should display error toast when api throws error', async () => { + getKnowledgeBaseIndicesMock.mockRejectedValue(new Error('this is an error')); + await act(async () => { + const { waitForNextUpdate } = renderHook(() => useKnowledgeBaseIndices(defaultProps)); + await waitForNextUpdate(); + + expect(toasts.addError).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.tsx new file mode 100644 index 0000000000000..2b245c70754b5 --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_indices.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseQueryResult } from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; +import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; +import type { IToasts } from '@kbn/core-notifications-browser'; +import { i18n } from '@kbn/i18n'; +import { GetKnowledgeBaseIndicesResponse } from '@kbn/elastic-assistant-common'; +import { getKnowledgeBaseIndices } from './api'; + +const KNOWLEDGE_BASE_INDICES_QUERY_KEY = ['elastic-assistant', 'knowledge-base-indices']; + +export interface UseKnowledgeBaseIndicesParams { + http: HttpSetup; + toasts?: IToasts; +} + +/** + * Hook for getting indices that have fields of `semantic_text` type. + * + * @param {Object} options - The options object. + * @param {HttpSetup} options.http - HttpSetup + * @param {IToasts} [options.toasts] - IToasts + * + * @returns {useQuery} hook for getting indices that have fields of `semantic_text` type + */ +export const useKnowledgeBaseIndices = ({ + http, + toasts, +}: UseKnowledgeBaseIndicesParams): UseQueryResult< + GetKnowledgeBaseIndicesResponse, + IHttpFetchError +> => { + return useQuery( + KNOWLEDGE_BASE_INDICES_QUERY_KEY, + async ({ signal }) => { + return getKnowledgeBaseIndices({ http, signal }); + }, + { + onError: (error: IHttpFetchError) => { + if (error.name !== 'AbortError') { + toasts?.addError( + error.body && error.body.message ? new Error(error.body.message) : error, + { + title: i18n.translate('xpack.elasticAssistant.knowledgeBase.indicesError', { + defaultMessage: 'Error fetching Knowledge Base Indices', + }), + } + ); + } + }, + } + ); +}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx index 4f85dae819f0f..cfc8d2d3d52f9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.test.tsx @@ -24,6 +24,7 @@ import { MOCK_QUICK_PROMPTS } from '../../mock/quick_prompt'; import { useAssistantContext } from '../../..'; import { I18nProvider } from '@kbn/i18n-react'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { useKnowledgeBaseIndices } from '../../assistant/api/knowledge_base/use_knowledge_base_indices'; const mockContext = { basePromptContexts: MOCK_QUICK_PROMPTS, @@ -44,13 +45,13 @@ jest.mock('../../assistant/api/knowledge_base/entries/use_update_knowledge_base_ jest.mock('../../assistant/api/knowledge_base/entries/use_delete_knowledge_base_entries'); jest.mock('../../assistant/settings/use_settings_updater/use_settings_updater'); +jest.mock('../../assistant/api/knowledge_base/use_knowledge_base_indices'); jest.mock('../../assistant/api/knowledge_base/use_knowledge_base_status'); jest.mock('../../assistant/api/knowledge_base/entries/use_knowledge_base_entries'); jest.mock( '../../assistant/common/components/assistant_settings_management/flyout/use_flyout_modal_visibility' ); const mockDataViews = { - getIndices: jest.fn().mockResolvedValue([{ name: 'index-1' }, { name: 'index-2' }]), getFieldsForWildcard: jest.fn().mockResolvedValue([ { name: 'field-1', esTypes: ['semantic_text'] }, { name: 'field-2', esTypes: ['text'] }, @@ -148,6 +149,9 @@ describe('KnowledgeBaseSettingsManagement', () => { }, isFetched: true, }); + (useKnowledgeBaseIndices as jest.Mock).mockReturnValue({ + data: { indices: ['index-1', 'index-2'] }, + }); (useKnowledgeBaseEntries as jest.Mock).mockReturnValue({ data: { data: mockData }, isFetching: false, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx index 092cc7e36689e..904ceba7a1f6f 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx @@ -434,6 +434,7 @@ export const KnowledgeBaseSettingsManagement: React.FC = React.memo(({ d /> ) : ( { const mockSetEntry = jest.fn(); const mockDataViews = { - getIndices: jest.fn().mockResolvedValue([{ name: 'index-1' }, { name: 'index-2' }]), getFieldsForWildcard: jest.fn().mockResolvedValue([ { name: 'field-1', esTypes: ['semantic_text'] }, { name: 'field-2', esTypes: ['text'] }, @@ -24,6 +27,9 @@ describe('IndexEntryEditor', () => { ]), getExistingIndices: jest.fn().mockResolvedValue(['index-1']), } as unknown as DataViewsContract; + const http = { + get: jest.fn(), + } as unknown as HttpSetup; const defaultProps = { dataViews: mockDataViews, @@ -37,10 +43,14 @@ describe('IndexEntryEditor', () => { queryDescription: 'Test Query Description', users: [], } as unknown as IndexEntry, + http, }; beforeEach(() => { jest.clearAllMocks(); + (useKnowledgeBaseIndices as jest.Mock).mockReturnValue({ + data: { indices: ['index-1', 'index-2'] }, + }); }); it('renders the form fields with initial values', async () => { @@ -102,7 +112,6 @@ describe('IndexEntryEditor', () => { const { getAllByTestId, getByTestId } = render(); await waitFor(() => { - expect(mockDataViews.getIndices).toHaveBeenCalled(); fireEvent.click(getByTestId('index-combobox')); fireEvent.click(getAllByTestId('comboBoxToggleListButton')[0]); fireEvent.click(getByTestId('index-2')); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx index dfc3cd0086686..b55fb4b1b8270 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index_entry_editor.tsx @@ -20,10 +20,13 @@ import useAsync from 'react-use/lib/useAsync'; import React, { useCallback, useMemo } from 'react'; import { IndexEntry } from '@kbn/elastic-assistant-common'; import { DataViewsContract } from '@kbn/data-views-plugin/public'; +import { HttpSetup } from '@kbn/core-http-browser'; import * as i18n from './translations'; import { isGlobalEntry } from './helpers'; +import { useKnowledgeBaseIndices } from '../../assistant/api/knowledge_base/use_knowledge_base_indices'; interface Props { + http: HttpSetup; dataViews: DataViewsContract; entry?: IndexEntry; originalEntry?: IndexEntry; @@ -32,7 +35,7 @@ interface Props { } export const IndexEntryEditor: React.FC = React.memo( - ({ dataViews, entry, setEntry, hasManageGlobalKnowledgeBase, originalEntry }) => { + ({ http, dataViews, entry, setEntry, hasManageGlobalKnowledgeBase, originalEntry }) => { const privateUsers = useMemo(() => { const originalUsers = originalEntry?.users; if (originalEntry && !isGlobalEntry(originalEntry)) { @@ -93,18 +96,16 @@ export const IndexEntryEditor: React.FC = React.memo( entry?.users?.length === 0 ? sharingOptions[1].value : sharingOptions[0].value; // Index - const indexOptions = useAsync(async () => { - const indices = await dataViews.getIndices({ - pattern: '*', - isRollupIndex: () => false, - }); - - return indices.map((index) => ({ - 'data-test-subj': index.name, - label: index.name, - value: index.name, + const { data: kbIndices } = useKnowledgeBaseIndices({ + http, + }); + const indexOptions = useMemo(() => { + return kbIndices?.indices.map((index) => ({ + 'data-test-subj': index, + label: index, + value: index, })); - }, [dataViews]); + }, [kbIndices?.indices]); const { value: isMissingIndex } = useAsync(async () => { if (!entry?.index?.length) return false; @@ -272,6 +273,7 @@ export const IndexEntryEditor: React.FC = React.memo( fullWidth isInvalid={isMissingIndex} error={isMissingIndex && <>{i18n.MISSING_INDEX_ERROR}} + helpText={i18n.ENTRY_INDEX_NAME_INPUT_DESCRIPTION} > = React.memo( singleSelection={{ asPlainText: true }} onCreateOption={onCreateIndexOption} fullWidth - options={indexOptions.value ?? []} + options={indexOptions ?? []} selectedOptions={ entry?.index ? [ diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/translations.ts index 98af0eabea6b5..24784586edcdf 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/translations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/translations.ts @@ -234,6 +234,14 @@ export const ENTRY_INDEX_NAME_INPUT_LABEL = i18n.translate( } ); +export const ENTRY_INDEX_NAME_INPUT_DESCRIPTION = i18n.translate( + 'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettingsManagement.entryIndexNameInputDescription', + { + defaultMessage: + 'Indices will only be available to select from this drop down list if they contain a semantic_text field. Please refer to the documentation for more information on configuring an index for use as a custom knowledge source.', + } +); + export const ENTRY_FIELD_INPUT_LABEL = i18n.translate( 'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettingsManagement.entryFieldInputLabel', { diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts index 9dc57bab25ef3..f6f3007c8f948 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request.ts @@ -23,6 +23,7 @@ import { ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_BY_ID_MESSAGES, ELASTIC_AI_ASSISTANT_CONVERSATIONS_URL_FIND, ELASTIC_AI_ASSISTANT_EVALUATE_URL, + ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, ELASTIC_AI_ASSISTANT_PROMPTS_URL_BULK_ACTION, ELASTIC_AI_ASSISTANT_PROMPTS_URL_FIND, @@ -46,6 +47,12 @@ export const requestMock = { create: httpServerMock.createKibanaRequest, }; +export const getGetKnowledgeBaseIndicesRequest = () => + requestMock.create({ + method: 'get', + path: ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, + }); + export const getGetKnowledgeBaseStatusRequest = (resource?: string) => requestMock.create({ method: 'get', diff --git a/x-pack/plugins/elastic_assistant/server/routes/index.ts b/x-pack/plugins/elastic_assistant/server/routes/index.ts index a6d7a4298c2b7..928c3211faa9b 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/index.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/index.ts @@ -14,6 +14,7 @@ export { getAttackDiscoveryRoute } from './attack_discovery/get/get_attack_disco // Knowledge Base export { deleteKnowledgeBaseRoute } from './knowledge_base/delete_knowledge_base'; +export { getKnowledgeBaseIndicesRoute } from './knowledge_base/get_knowledge_base_indices'; export { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; export { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.test.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.test.ts new file mode 100644 index 0000000000000..e7eaa75407248 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getKnowledgeBaseIndicesRoute } from './get_knowledge_base_indices'; +import { serverMock } from '../../__mocks__/server'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { getGetKnowledgeBaseIndicesRequest } from '../../__mocks__/request'; + +const mockFieldCaps = { + indices: [ + '.ds-logs-endpoint.alerts-default-2024.10.31-000001', + '.ds-metrics-endpoint.metadata-default-2024.10.31-000001', + '.internal.alerts-security.alerts-default-000001', + 'metrics-endpoint.metadata_current_default', + 'semantic-index-1', + 'semantic-index-2', + ], + fields: { + content: { + unmapped: { + type: 'unmapped', + metadata_field: false, + searchable: false, + aggregatable: false, + indices: [ + '.ds-logs-endpoint.alerts-default-2024.10.31-000001', + '.ds-metrics-endpoint.metadata-default-2024.10.31-000001', + '.internal.alerts-security.alerts-default-000001', + 'metrics-endpoint.metadata_current_default', + ], + }, + semantic_text: { + type: 'semantic_text', + metadata_field: false, + searchable: true, + aggregatable: false, + indices: ['semantic-index-1', 'semantic-index-2'], + }, + }, + }, +}; + +describe('Get Knowledge Base Status Route', () => { + let server: ReturnType; + + let { context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ context } = requestContextMock.createTools()); + context.core.elasticsearch.client.asCurrentUser.fieldCaps.mockResponse(mockFieldCaps); + + getKnowledgeBaseIndicesRoute(server.router); + }); + + describe('Status codes', () => { + test('returns 200 and all indices with `semantic_text` type fields', async () => { + const response = await server.inject( + getGetKnowledgeBaseIndicesRequest(), + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + indices: ['semantic-index-1', 'semantic-index-2'], + }); + expect(context.core.elasticsearch.client.asCurrentUser.fieldCaps).toBeCalledWith({ + index: '*', + fields: '*', + types: ['semantic_text'], + include_unmapped: true, + }); + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.ts b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.ts new file mode 100644 index 0000000000000..18191291468de --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/routes/knowledge_base/get_knowledge_base_indices.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; + +import { + ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, + GetKnowledgeBaseIndicesResponse, +} from '@kbn/elastic-assistant-common'; +import { IKibanaResponse } from '@kbn/core/server'; +import { buildResponse } from '../../lib/build_response'; +import { ElasticAssistantPluginRouter } from '../../types'; + +/** + * Get the indices that have fields of `sematic_text` type + * + * @param router IRouter for registering routes + */ +export const getKnowledgeBaseIndicesRoute = (router: ElasticAssistantPluginRouter) => { + router.versioned + .get({ + access: 'internal', + path: ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, + options: { + tags: ['access:elasticAssistant'], + }, + }) + .addVersion( + { + version: ELASTIC_AI_ASSISTANT_INTERNAL_API_VERSION, + validate: false, + }, + async (context, _, response): Promise> => { + const resp = buildResponse(response); + const ctx = await context.resolve(['core', 'elasticAssistant', 'licensing']); + const logger = ctx.elasticAssistant.logger; + const esClient = ctx.core.elasticsearch.client.asCurrentUser; + + try { + const body: GetKnowledgeBaseIndicesResponse = { + indices: [], + }; + + const res = await esClient.fieldCaps({ + index: '*', + fields: '*', + types: ['semantic_text'], + include_unmapped: true, + }); + + const indices = res.fields.content?.semantic_text?.indices; + if (indices) { + body.indices = Array.isArray(indices) ? indices : [indices]; + } + + return response.ok({ body }); + } catch (err) { + logger.error(err); + const error = transformError(err); + + return resp.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts index 7898629e15b5c..d722e31cb2338 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/register_routes.ts @@ -18,6 +18,7 @@ import { updateConversationRoute } from './user_conversations/update_route'; import { findUserConversationsRoute } from './user_conversations/find_route'; import { bulkActionConversationsRoute } from './user_conversations/bulk_actions_route'; import { appendConversationMessageRoute } from './user_conversations/append_conversation_messages_route'; +import { getKnowledgeBaseIndicesRoute } from './knowledge_base/get_knowledge_base_indices'; import { getKnowledgeBaseStatusRoute } from './knowledge_base/get_knowledge_base_status'; import { postKnowledgeBaseRoute } from './knowledge_base/post_knowledge_base'; import { getEvaluateRoute } from './evaluate/get_evaluate'; @@ -60,6 +61,7 @@ export const registerRoutes = ( findUserConversationsRoute(router); // Knowledge Base Setup + getKnowledgeBaseIndicesRoute(router); getKnowledgeBaseStatusRoute(router); postKnowledgeBaseRoute(router); diff --git a/x-pack/test/functional/es_archives/security_solution/semantic_text_fields/data.json b/x-pack/test/functional/es_archives/security_solution/semantic_text_fields/data.json new file mode 100644 index 0000000000000..3cef2aae8ff4f --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/semantic_text_fields/data.json @@ -0,0 +1,161 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "semantic_text_fields", + "source": { + "@timestamp": "2024-11-01T15:52:16.648Z", + "content": { + "text": "my favorite color is green", + "inference": { + "inference_id": "elastic-security-ai-assistant-elser2", + "model_settings": { + "task_type": "sparse_embedding" + }, + "chunks": [ + { + "text": "my favorite color is green", + "embeddings": { + "green": 2.8714406, + "favorite": 2.5192127, + "color": 2.499853, + "favourite": 1.7829537, + "colors": 1.2280636, + "my": 1.1292906, + "friend": 0.87358737, + "rainbow": 0.8518238, + "love": 0.8146304, + "choice": 0.7517174, + "nature": 0.62242556, + "beautiful": 0.6110072, + "personality": 0.5559894, + "dr": 0.5296162, + "your": 0.51745296, + "art": 0.45324937, + "colour": 0.44607934, + "theme": 0.4360909, + "mood": 0.43253413, + "personal": 0.4201024, + "style": 0.39435387, + "blue": 0.38090202, + "nickname": 0.37952134, + "design": 0.37043664, + "dream": 0.3620103, + "desire": 0.35553402, + "best": 0.32577398, + "favorites": 0.30795538, + "humor": 0.30244058, + "popular": 0.2957705, + "brand": 0.28912684, + "neutral": 0.28545624, + "passion": 0.28457505, + "i": 0.27936152, + "preference": 0.24133624, + "inspiration": 0.24008423, + "purple": 0.23559056, + "culture": 0.23260204, + "flower": 0.21190192, + "bright": 0.20443156, + "beauty": 0.20076275, + "aura": 0.19355631, + "palette": 0.17414959, + "wonder": 0.16287619, + "photo": 0.16179858, + "orange": 0.14167522, + "dress": 0.12800644, + "camouflage": 0.061010167, + "grass": 0.05907971, + "tone": 0.028165601, + "painting": 0.026917756, + "cartoon": 0.019969255, + "always": 0.013872984, + "yellow": 0.0113299545, + "colorful": 0.0036836881 + } + } + ] + } + }, + "text": "my favorite color is green" + } + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "semantic_text_fields", + "source": { + "@timestamp": "2024-11-01T15:59:34.610Z", + "content": { + "text": "my favorite food is pasta", + "inference": { + "inference_id": "elastic-security-ai-assistant-elser2", + "model_settings": { + "task_type": "sparse_embedding" + }, + "chunks": [ + { + "text": "my favorite food is pasta", + "embeddings": { + "pasta": 2.8800304, + "favorite": 2.604094, + "food": 2.3218846, + "favourite": 1.9555497, + "foods": 1.8483086, + "my": 1.2514912, + "italian": 1.1385239, + "noodles": 0.9831485, + "ravi": 0.8951362, + "friend": 0.73949015, + "popular": 0.7100698, + "rich": 0.6932188, + "diet": 0.6634609, + "love": 0.64316577, + "famous": 0.5988269, + "eat": 0.5932935, + "your": 0.5730934, + "flavor": 0.54839855, + "i": 0.54047567, + "choice": 0.52978134, + "taste": 0.5071018, + "personal": 0.49576512, + "favorites": 0.3890845, + "soup": 0.37032336, + "rome": 0.31307018, + "list": 0.28725958, + "rice": 0.28322402, + "dinner": 0.26971146, + "familiar": 0.22095734, + "vegetable": 0.21891576, + "bland": 0.20330611, + "obsession": 0.19760907, + "latin": 0.19196871, + "drink": 0.18664016, + "culture": 0.18313695, + "shop": 0.18128464, + "style": 0.16871399, + "greek": 0.16838282, + "our": 0.1634054, + "is": 0.14557752, + "category": 0.12824115, + "chef": 0.11436942, + "italy": 0.095480226, + "family": 0.0816377, + "classic": 0.064411946, + "best": 0.05733679, + "always": 0.054197904, + "type": 0.0500416, + "lover": 0.037193555, + "bread": 0.031606384, + "fruit": 0.00062303204 + } + } + ] + } + }, + "text": "my favorite food is pasta" + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/semantic_text_fields/mappings.json b/x-pack/test/functional/es_archives/security_solution/semantic_text_fields/mappings.json new file mode 100644 index 0000000000000..7c5deef42aafb --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/semantic_text_fields/mappings.json @@ -0,0 +1,30 @@ +{ + "type": "index", + "value": { + "index": "semantic_text_fields", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "content": { + "type": "semantic_text", + "inference_id": "elastic-security-ai-assistant-elser2", + "model_settings": { + "task_type": "sparse_embedding" + } + }, + "text": { + "type": "text" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/index.ts index 21469b8e67606..8aea7b00f9eed 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/index.ts @@ -19,5 +19,6 @@ export default function ({ loadTestFile, getService }: FtrProviderContext) { }); loadTestFile(require.resolve('./entries')); + loadTestFile(require.resolve('./semntic_text_indices')); }); } diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/semntic_text_indices.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/semntic_text_indices.ts new file mode 100644 index 0000000000000..d983ed79b97ad --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/semntic_text_indices.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { FtrProviderContext } from '../../../../../ftr_provider_context'; +import { getKnowledgeBaseIndices } from '../utils/helpers'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const esArchiver = getService('esArchiver'); + + describe('@ess Security AI Assistant - Indices with `semantic_text` fields', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ignore_fields'); + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/semantic_text_fields' + ); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ignore_fields'); + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/semantic_text_fields' + ); + }); + + it('should return all existing indices with `semantic_text` fields', async () => { + const indices = await getKnowledgeBaseIndices({ supertest, log }); + + expect(indices).toEqual({ indices: ['semantic_text_fields'] }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/utils/helpers.ts b/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/utils/helpers.ts index cdbb1cca4de24..36b2963f5b538 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/utils/helpers.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/utils/helpers.ts @@ -8,7 +8,9 @@ import { Client } from '@elastic/elasticsearch'; import { CreateKnowledgeBaseResponse, + ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL, ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_URL, + GetKnowledgeBaseIndicesResponse, } from '@kbn/elastic-assistant-common'; import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; @@ -97,3 +99,31 @@ export const clearKnowledgeBase = async (es: Client, space = 'default') => { refresh: true, }); }; + +/** + * Get indices with the `semantic_text` type fields + * @param supertest The supertest deps + * @param log The tooling logger + */ +export const getKnowledgeBaseIndices = async ({ + supertest, + log, +}: { + supertest: SuperTest.Agent; + log: ToolingLog; +}): Promise => { + const response = await supertest + .get(ELASTIC_AI_ASSISTANT_KNOWLEDGE_BASE_INDICES_URL) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .send(); + if (response.status !== 200) { + throw new Error( + `Unexpected non 200 ok when attempting to find entries: ${JSON.stringify( + response.status + )},${JSON.stringify(response, null, 4)}` + ); + } else { + return response.body; + } +};