diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js index e19b2cbceda33..676830e94a280 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js @@ -360,9 +360,6 @@ export class FileDataVisualizerView extends Component { fileName={fileName} fileContents={fileContents} data={data} - dataViewsContract={this.props.dataViewsContract} - dataStart={this.props.dataStart} - fileUpload={this.props.fileUpload} getAdditionalLinks={this.props.getAdditionalLinks} resultLinks={this.props.resultLinks} capabilities={this.props.capabilities} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx index 53b92ab1f6414..5a6db4f544fd9 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx @@ -31,6 +31,8 @@ export interface Statuses { createDataView: boolean; createPipeline: boolean; permissionCheckStatus: IMPORT_STATUS; + initializeDeployment: boolean; + initializeDeploymentStatus: IMPORT_STATUS; } export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { @@ -45,6 +47,8 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { uploadStatus, createDataView, createPipeline, + initializeDeployment, + initializeDeploymentStatus, } = statuses; let statusInfo = null; @@ -58,27 +62,37 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { ) { completedStep = 0; } + + if ( + readStatus === IMPORT_STATUS.COMPLETE && + initializeDeployment === true && + initializeDeploymentStatus === IMPORT_STATUS.INCOMPLETE + ) { + completedStep = 1; + } + if ( readStatus === IMPORT_STATUS.COMPLETE && + (initializeDeployment === false || initializeDeploymentStatus === IMPORT_STATUS.COMPLETE) && indexCreatedStatus === IMPORT_STATUS.INCOMPLETE && ingestPipelineCreatedStatus === IMPORT_STATUS.INCOMPLETE ) { - completedStep = 1; + completedStep = 2; } if (indexCreatedStatus === IMPORT_STATUS.COMPLETE) { - completedStep = 2; + completedStep = 3; } if ( ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE || (createPipeline === false && indexCreatedStatus === IMPORT_STATUS.COMPLETE) ) { - completedStep = 3; + completedStep = 4; } if (uploadStatus === IMPORT_STATUS.COMPLETE) { - completedStep = 4; + completedStep = 5; } if (dataViewCreatedStatus === IMPORT_STATUS.COMPLETE) { - completedStep = 5; + completedStep = 6; } let processFileTitle = i18n.translate( @@ -87,6 +101,12 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { defaultMessage: 'Process file', } ); + let initializeDeploymentTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.initializeDeploymentTitle', + { + defaultMessage: 'Initialize model deployment', + } + ); let createIndexTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.createIndexTitle', { @@ -146,13 +166,43 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {

); } - if (completedStep >= 1) { + if (initializeDeployment) { + if (completedStep >= 1) { + processFileTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.fileProcessedTitle', + { + defaultMessage: 'File processed', + } + ); + initializeDeploymentTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.initializingDeploymentTitle', + { + defaultMessage: 'Initializing model deployment', + } + ); + statusInfo = ( +

+ +

+ ); + } + } + if (completedStep >= 2) { processFileTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.fileProcessedTitle', { defaultMessage: 'File processed', } ); + initializeDeploymentTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.deploymentInitializedTitle', + { + defaultMessage: 'Model deployed', + } + ); createIndexTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.creatingIndexTitle', { @@ -162,7 +212,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { statusInfo = createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus; } - if (completedStep >= 2) { + if (completedStep >= 3) { createIndexTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.indexCreatedTitle', { @@ -178,7 +228,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { statusInfo = createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus; } - if (completedStep >= 3) { + if (completedStep >= 4) { createIngestPipelineTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.ingestPipelineCreatedTitle', { @@ -193,7 +243,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { ); statusInfo = ; } - if (completedStep >= 4) { + if (completedStep >= 5) { uploadingDataTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.dataUploadedTitle', { @@ -219,7 +269,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { statusInfo = null; } } - if (completedStep >= 5) { + if (completedStep >= 6) { createDataViewTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.dataViewCreatedTitle', { @@ -239,44 +289,58 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { : 'selected') as EuiStepStatus, onClick: () => {}, }, - { - title: createIndexTitle, - status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first - ? indexCreatedStatus + ]; + + if (initializeDeployment === true) { + steps.push({ + title: initializeDeploymentTitle, + status: (initializeDeploymentStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first + ? initializeDeploymentStatus : completedStep === 1 // Then show selected/incomplete states ? 'selected' : 'incomplete') as EuiStepStatus, onClick: () => {}, - }, - { - title: uploadingDataTitle, - status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first - ? uploadStatus - : completedStep === 3 // Then show selected/incomplete states - ? 'selected' - : 'incomplete') as EuiStepStatus, - onClick: () => {}, - }, - ]; + }); + } + + steps.push({ + title: createIndexTitle, + status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first + ? indexCreatedStatus + : completedStep === 2 // Then show selected/incomplete states + ? 'selected' + : 'incomplete') as EuiStepStatus, + onClick: () => {}, + }); if (createPipeline === true) { - steps.splice(2, 0, { + steps.push({ title: createIngestPipelineTitle, status: (ingestPipelineCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first ? ingestPipelineCreatedStatus - : completedStep === 2 // Then show selected/incomplete states + : completedStep === 3 // Then show selected/incomplete states ? 'selected' : 'incomplete') as EuiStepStatus, onClick: () => {}, }); } + steps.push({ + title: uploadingDataTitle, + status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first + ? uploadStatus + : completedStep === 4 // Then show selected/incomplete states + ? 'selected' + : 'incomplete') as EuiStepStatus, + onClick: () => {}, + }); + if (createDataView === true) { steps.push({ title: createDataViewTitle, status: (dataViewCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first ? dataViewCreatedStatus - : completedStep === 4 // Then show selected/incomplete states + : completedStep === 5 // Then show selected/incomplete states ? 'selected' : 'incomplete') as EuiStepStatus, onClick: () => {}, @@ -284,21 +348,21 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { } return ( - + <> {statusInfo && ( - + <> {statusInfo} - + )} - + ); }; const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => { return ( - + <>

= ({ progress }) => { />

{progress < 100 && ( - + <> - + )} -
+ ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts new file mode 100644 index 0000000000000..a402d203585d2 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceInferenceEndpointInfo } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { HttpSetup } from '@kbn/core/public'; + +const POLL_INTERVAL = 5; // seconds + +export class AutoDeploy { + private inferError: Error | null = null; + constructor(private readonly http: HttpSetup, private readonly inferenceId: string) {} + + public async deploy() { + this.inferError = null; + if (await this.isDeployed()) { + return; + } + + this.infer().catch((e) => { + // ignore timeout errors + // The deployment may take a long time + // we'll know when it's ready from polling the inference endpoints + // looking for num_allocations + const status = e.response?.status; + if (status === 408 || status === 504 || status === 502) { + return; + } + this.inferError = e; + }); + await this.pollIsDeployed(); + } + + private async infer() { + return this.http.fetch( + `/internal/data_visualizer/inference/${this.inferenceId}`, + { + method: 'POST', + version: '1', + body: JSON.stringify({ input: '' }), + } + ); + } + + private async isDeployed() { + const inferenceEndpoints = await this.http.fetch( + '/internal/data_visualizer/inference_endpoints', + { + method: 'GET', + version: '1', + } + ); + return inferenceEndpoints.some((endpoint) => { + return ( + endpoint.inference_id === this.inferenceId && endpoint.service_settings.num_allocations > 0 + ); + }); + } + + private async pollIsDeployed() { + while (true) { + if (this.inferError !== null) { + throw this.inferError; + } + const isDeployed = await this.isDeployed(); + if (isDeployed) { + // break out of the loop once we have a successful deployment + return; + } + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL * 1000)); + } + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts index 8611e1ccd2cab..a44cb4fb890fe 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts @@ -12,13 +12,17 @@ import type { } from '@kbn/file-upload-plugin/common/types'; import type { FileUploadStartApi } from '@kbn/file-upload-plugin/public/api'; import { i18n } from '@kbn/i18n'; +import type { HttpSetup } from '@kbn/core/public'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { IMPORT_STATUS } from '../import_progress/import_progress'; +import { AutoDeploy } from './auto_deploy'; interface Props { data: ArrayBuffer; results: FindFileStructureResponse; dataViewsContract: DataViewsServicePublic; fileUpload: FileUploadStartApi; + http: HttpSetup; } interface Config { @@ -33,7 +37,7 @@ interface Config { } export async function importData(props: Props, config: Config, setState: (state: unknown) => void) { - const { data, results, dataViewsContract, fileUpload } = props; + const { data, results, dataViewsContract, fileUpload, http } = props; const { index, dataView, @@ -76,14 +80,6 @@ export async function importData(props: Props, config: Config, setState: (state: return; } - setState({ - importing: true, - imported: false, - reading: true, - initialized: true, - permissionCheckStatus: IMPORT_STATUS.COMPLETE, - }); - let success = true; let settings = {}; @@ -122,7 +118,15 @@ export async function importData(props: Props, config: Config, setState: (state: errors.push(`${parseError} ${error.message}`); } + const inferenceId = getInferenceId(mappings); + setState({ + importing: true, + imported: false, + reading: true, + initialized: true, + permissionCheckStatus: IMPORT_STATUS.COMPLETE, + initializeDeployment: inferenceId !== null, parseJSONStatus: getSuccess(success), }); @@ -147,6 +151,32 @@ export async function importData(props: Props, config: Config, setState: (state: return; } + if (inferenceId) { + // Initialize deployment + const autoDeploy = new AutoDeploy(http, inferenceId); + + try { + await autoDeploy.deploy(); + setState({ + initializeDeploymentStatus: IMPORT_STATUS.COMPLETE, + }); + } catch (error) { + success = false; + const deployError = i18n.translate('xpack.dataVisualizer.file.importView.deployModelError', { + defaultMessage: 'Error deploying trained model:', + }); + errors.push(`${deployError} ${error.message}`); + setState({ + initializeDeploymentStatus: IMPORT_STATUS.FAILED, + errors, + }); + } + } + + if (success === false) { + return; + } + const initializeImportResp = await importer.initializeImport(index, settings, mappings, pipeline); const timeFieldName = importer.getTimeField(); @@ -245,3 +275,12 @@ async function createKibanaDataView( function getSuccess(success: boolean) { return success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED; } + +function getInferenceId(mappings: MappingTypeMapping) { + for (const value of Object.values(mappings.properties ?? {})) { + if (value.type === 'semantic_text') { + return value.inference_id; + } + } + return null; +} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js index 8481ed76e5654..3f30062270060 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js @@ -21,6 +21,7 @@ import { } from '@elastic/eui'; import { debounce } from 'lodash'; +import { context } from '@kbn/kibana-react-plugin/public'; import { ResultsLinks } from '../../../common/components/results_links'; import { FilebeatConfigFlyout } from '../../../common/components/filebeat_config_flyout'; import { ImportProgress, IMPORT_STATUS } from '../import_progress'; @@ -76,14 +77,17 @@ const DEFAULT_STATE = { combinedFields: [], importer: undefined, createPipeline: true, + initializeDeployment: false, + initializeDeploymentStatus: IMPORT_STATUS.INCOMPLETE, }; export class ImportView extends Component { + static contextType = context; + constructor(props) { super(props); this.state = getDefaultState(DEFAULT_STATE, this.props.results, this.props.capabilities); - this.dataViewsContract = props.dataViewsContract; } componentDidMount() { @@ -98,7 +102,12 @@ export class ImportView extends Component { }; clickImport = () => { - const { data, results, dataViewsContract, fileUpload } = this.props; + const { data, results } = this.props; + const { + data: { dataViews: dataViewsContract }, + fileUpload, + http, + } = this.context.services; const { index, dataView, @@ -110,12 +119,13 @@ export class ImportView extends Component { } = this.state; const createPipeline = pipelineString !== ''; + this.setState({ createPipeline, }); importData( - { data, results, dataViewsContract, fileUpload }, + { data, results, dataViewsContract, fileUpload, http }, { index, dataView, @@ -150,7 +160,7 @@ export class ImportView extends Component { return; } - const exists = await this.props.fileUpload.checkIndexExists(index); + const exists = await this.context.services.fileUpload.checkIndexExists(index); const indexNameError = exists ? ( )} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx index 01d9d2c37194f..6289a73e2a664 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx @@ -44,7 +44,6 @@ export const FileDataVisualizer: FC = ({ getAdditionalLinks, resultLinks , logger: Logger) const filteredInferenceEndpoints = endpoints.filter((endpoint) => { return ( - (endpoint.task_type === 'sparse_embedding' || - endpoint.task_type === 'text_embedding') && - endpoint.service_settings.num_allocations >= 0 + endpoint.task_type === 'sparse_embedding' || endpoint.task_type === 'text_embedding' ); }); @@ -108,4 +106,50 @@ export function routes(coreSetup: CoreSetup, logger: Logger) } } ); + + /** + * @apiGroup DataVisualizer + * + * @api {get} /internal/data_visualizer/inference/{inferenceId} Runs inference on a given inference endpoint with the provided input + * @apiName inference + * @apiDescription Runs inference on a given inference endpoint with the provided input. + */ + router.versioned + .post({ + path: '/internal/data_visualizer/inference/{inferenceId}', + access: 'internal', + options: { + tags: ['access:fileUpload:analyzeFile'], + }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: schema.object({ + inferenceId: schema.string(), + }), + body: schema.object({ + input: schema.string(), + }), + }, + }, + }, + async (context, request, response) => { + try { + const inferenceId = request.params.inferenceId; + const input = request.body.input; + const esClient = (await context.core).elasticsearch.client; + const body = await esClient.asCurrentUser.inference.inference({ + inference_id: inferenceId, + input, + }); + + return response.ok({ body }); + } catch (e) { + return response.customError(wrapError(e)); + } + } + ); }