diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js
index e19b2cbceda33..676830e94a280 100644
--- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js
+++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js
@@ -360,9 +360,6 @@ export class FileDataVisualizerView extends Component {
fileName={fileName}
fileContents={fileContents}
data={data}
- dataViewsContract={this.props.dataViewsContract}
- dataStart={this.props.dataStart}
- fileUpload={this.props.fileUpload}
getAdditionalLinks={this.props.getAdditionalLinks}
resultLinks={this.props.resultLinks}
capabilities={this.props.capabilities}
diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx
index 53b92ab1f6414..5a6db4f544fd9 100644
--- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx
+++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx
@@ -31,6 +31,8 @@ export interface Statuses {
createDataView: boolean;
createPipeline: boolean;
permissionCheckStatus: IMPORT_STATUS;
+ initializeDeployment: boolean;
+ initializeDeploymentStatus: IMPORT_STATUS;
}
export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
@@ -45,6 +47,8 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
uploadStatus,
createDataView,
createPipeline,
+ initializeDeployment,
+ initializeDeploymentStatus,
} = statuses;
let statusInfo = null;
@@ -58,27 +62,37 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
) {
completedStep = 0;
}
+
+ if (
+ readStatus === IMPORT_STATUS.COMPLETE &&
+ initializeDeployment === true &&
+ initializeDeploymentStatus === IMPORT_STATUS.INCOMPLETE
+ ) {
+ completedStep = 1;
+ }
+
if (
readStatus === IMPORT_STATUS.COMPLETE &&
+ (initializeDeployment === false || initializeDeploymentStatus === IMPORT_STATUS.COMPLETE) &&
indexCreatedStatus === IMPORT_STATUS.INCOMPLETE &&
ingestPipelineCreatedStatus === IMPORT_STATUS.INCOMPLETE
) {
- completedStep = 1;
+ completedStep = 2;
}
if (indexCreatedStatus === IMPORT_STATUS.COMPLETE) {
- completedStep = 2;
+ completedStep = 3;
}
if (
ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE ||
(createPipeline === false && indexCreatedStatus === IMPORT_STATUS.COMPLETE)
) {
- completedStep = 3;
+ completedStep = 4;
}
if (uploadStatus === IMPORT_STATUS.COMPLETE) {
- completedStep = 4;
+ completedStep = 5;
}
if (dataViewCreatedStatus === IMPORT_STATUS.COMPLETE) {
- completedStep = 5;
+ completedStep = 6;
}
let processFileTitle = i18n.translate(
@@ -87,6 +101,12 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
defaultMessage: 'Process file',
}
);
+ let initializeDeploymentTitle = i18n.translate(
+ 'xpack.dataVisualizer.file.importProgress.initializeDeploymentTitle',
+ {
+ defaultMessage: 'Initialize model deployment',
+ }
+ );
let createIndexTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.createIndexTitle',
{
@@ -146,13 +166,43 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
);
}
- if (completedStep >= 1) {
+ if (initializeDeployment) {
+ if (completedStep >= 1) {
+ processFileTitle = i18n.translate(
+ 'xpack.dataVisualizer.file.importProgress.fileProcessedTitle',
+ {
+ defaultMessage: 'File processed',
+ }
+ );
+ initializeDeploymentTitle = i18n.translate(
+ 'xpack.dataVisualizer.file.importProgress.initializingDeploymentTitle',
+ {
+ defaultMessage: 'Initializing model deployment',
+ }
+ );
+ statusInfo = (
+
+
+
+ );
+ }
+ }
+ if (completedStep >= 2) {
processFileTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.fileProcessedTitle',
{
defaultMessage: 'File processed',
}
);
+ initializeDeploymentTitle = i18n.translate(
+ 'xpack.dataVisualizer.file.importProgress.deploymentInitializedTitle',
+ {
+ defaultMessage: 'Model deployed',
+ }
+ );
createIndexTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.creatingIndexTitle',
{
@@ -162,7 +212,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
statusInfo =
createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus;
}
- if (completedStep >= 2) {
+ if (completedStep >= 3) {
createIndexTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.indexCreatedTitle',
{
@@ -178,7 +228,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
statusInfo =
createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus;
}
- if (completedStep >= 3) {
+ if (completedStep >= 4) {
createIngestPipelineTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.ingestPipelineCreatedTitle',
{
@@ -193,7 +243,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
);
statusInfo = ;
}
- if (completedStep >= 4) {
+ if (completedStep >= 5) {
uploadingDataTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.dataUploadedTitle',
{
@@ -219,7 +269,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
statusInfo = null;
}
}
- if (completedStep >= 5) {
+ if (completedStep >= 6) {
createDataViewTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.dataViewCreatedTitle',
{
@@ -239,44 +289,58 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
: 'selected') as EuiStepStatus,
onClick: () => {},
},
- {
- title: createIndexTitle,
- status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
- ? indexCreatedStatus
+ ];
+
+ if (initializeDeployment === true) {
+ steps.push({
+ title: initializeDeploymentTitle,
+ status: (initializeDeploymentStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
+ ? initializeDeploymentStatus
: completedStep === 1 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
- },
- {
- title: uploadingDataTitle,
- status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
- ? uploadStatus
- : completedStep === 3 // Then show selected/incomplete states
- ? 'selected'
- : 'incomplete') as EuiStepStatus,
- onClick: () => {},
- },
- ];
+ });
+ }
+
+ steps.push({
+ title: createIndexTitle,
+ status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
+ ? indexCreatedStatus
+ : completedStep === 2 // Then show selected/incomplete states
+ ? 'selected'
+ : 'incomplete') as EuiStepStatus,
+ onClick: () => {},
+ });
if (createPipeline === true) {
- steps.splice(2, 0, {
+ steps.push({
title: createIngestPipelineTitle,
status: (ingestPipelineCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? ingestPipelineCreatedStatus
- : completedStep === 2 // Then show selected/incomplete states
+ : completedStep === 3 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
});
}
+ steps.push({
+ title: uploadingDataTitle,
+ status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
+ ? uploadStatus
+ : completedStep === 4 // Then show selected/incomplete states
+ ? 'selected'
+ : 'incomplete') as EuiStepStatus,
+ onClick: () => {},
+ });
+
if (createDataView === true) {
steps.push({
title: createDataViewTitle,
status: (dataViewCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? dataViewCreatedStatus
- : completedStep === 4 // Then show selected/incomplete states
+ : completedStep === 5 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
@@ -284,21 +348,21 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
}
return (
-
+ <>
{statusInfo && (
-
+ <>
{statusInfo}
-
+ >
)}
-
+ >
);
};
const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => {
return (
-
+ <>
= ({ progress }) => {
/>
{progress < 100 && (
-
+ <>
-
+ >
)}
-
+ >
);
};
diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts
new file mode 100644
index 0000000000000..a402d203585d2
--- /dev/null
+++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { InferenceInferenceEndpointInfo } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { HttpSetup } from '@kbn/core/public';
+
+const POLL_INTERVAL = 5; // seconds
+
+export class AutoDeploy {
+ private inferError: Error | null = null;
+ constructor(private readonly http: HttpSetup, private readonly inferenceId: string) {}
+
+ public async deploy() {
+ this.inferError = null;
+ if (await this.isDeployed()) {
+ return;
+ }
+
+ this.infer().catch((e) => {
+ // ignore timeout errors
+ // The deployment may take a long time
+ // we'll know when it's ready from polling the inference endpoints
+ // looking for num_allocations
+ const status = e.response?.status;
+ if (status === 408 || status === 504 || status === 502) {
+ return;
+ }
+ this.inferError = e;
+ });
+ await this.pollIsDeployed();
+ }
+
+ private async infer() {
+ return this.http.fetch(
+ `/internal/data_visualizer/inference/${this.inferenceId}`,
+ {
+ method: 'POST',
+ version: '1',
+ body: JSON.stringify({ input: '' }),
+ }
+ );
+ }
+
+ private async isDeployed() {
+ const inferenceEndpoints = await this.http.fetch(
+ '/internal/data_visualizer/inference_endpoints',
+ {
+ method: 'GET',
+ version: '1',
+ }
+ );
+ return inferenceEndpoints.some((endpoint) => {
+ return (
+ endpoint.inference_id === this.inferenceId && endpoint.service_settings.num_allocations > 0
+ );
+ });
+ }
+
+ private async pollIsDeployed() {
+ while (true) {
+ if (this.inferError !== null) {
+ throw this.inferError;
+ }
+ const isDeployed = await this.isDeployed();
+ if (isDeployed) {
+ // break out of the loop once we have a successful deployment
+ return;
+ }
+ await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL * 1000));
+ }
+ }
+}
diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts
index 8611e1ccd2cab..a44cb4fb890fe 100644
--- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts
+++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts
@@ -12,13 +12,17 @@ import type {
} from '@kbn/file-upload-plugin/common/types';
import type { FileUploadStartApi } from '@kbn/file-upload-plugin/public/api';
import { i18n } from '@kbn/i18n';
+import type { HttpSetup } from '@kbn/core/public';
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { IMPORT_STATUS } from '../import_progress/import_progress';
+import { AutoDeploy } from './auto_deploy';
interface Props {
data: ArrayBuffer;
results: FindFileStructureResponse;
dataViewsContract: DataViewsServicePublic;
fileUpload: FileUploadStartApi;
+ http: HttpSetup;
}
interface Config {
@@ -33,7 +37,7 @@ interface Config {
}
export async function importData(props: Props, config: Config, setState: (state: unknown) => void) {
- const { data, results, dataViewsContract, fileUpload } = props;
+ const { data, results, dataViewsContract, fileUpload, http } = props;
const {
index,
dataView,
@@ -76,14 +80,6 @@ export async function importData(props: Props, config: Config, setState: (state:
return;
}
- setState({
- importing: true,
- imported: false,
- reading: true,
- initialized: true,
- permissionCheckStatus: IMPORT_STATUS.COMPLETE,
- });
-
let success = true;
let settings = {};
@@ -122,7 +118,15 @@ export async function importData(props: Props, config: Config, setState: (state:
errors.push(`${parseError} ${error.message}`);
}
+ const inferenceId = getInferenceId(mappings);
+
setState({
+ importing: true,
+ imported: false,
+ reading: true,
+ initialized: true,
+ permissionCheckStatus: IMPORT_STATUS.COMPLETE,
+ initializeDeployment: inferenceId !== null,
parseJSONStatus: getSuccess(success),
});
@@ -147,6 +151,32 @@ export async function importData(props: Props, config: Config, setState: (state:
return;
}
+ if (inferenceId) {
+ // Initialize deployment
+ const autoDeploy = new AutoDeploy(http, inferenceId);
+
+ try {
+ await autoDeploy.deploy();
+ setState({
+ initializeDeploymentStatus: IMPORT_STATUS.COMPLETE,
+ });
+ } catch (error) {
+ success = false;
+ const deployError = i18n.translate('xpack.dataVisualizer.file.importView.deployModelError', {
+ defaultMessage: 'Error deploying trained model:',
+ });
+ errors.push(`${deployError} ${error.message}`);
+ setState({
+ initializeDeploymentStatus: IMPORT_STATUS.FAILED,
+ errors,
+ });
+ }
+ }
+
+ if (success === false) {
+ return;
+ }
+
const initializeImportResp = await importer.initializeImport(index, settings, mappings, pipeline);
const timeFieldName = importer.getTimeField();
@@ -245,3 +275,12 @@ async function createKibanaDataView(
function getSuccess(success: boolean) {
return success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED;
}
+
+function getInferenceId(mappings: MappingTypeMapping) {
+ for (const value of Object.values(mappings.properties ?? {})) {
+ if (value.type === 'semantic_text') {
+ return value.inference_id;
+ }
+ }
+ return null;
+}
diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js
index 8481ed76e5654..3f30062270060 100644
--- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js
+++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js
@@ -21,6 +21,7 @@ import {
} from '@elastic/eui';
import { debounce } from 'lodash';
+import { context } from '@kbn/kibana-react-plugin/public';
import { ResultsLinks } from '../../../common/components/results_links';
import { FilebeatConfigFlyout } from '../../../common/components/filebeat_config_flyout';
import { ImportProgress, IMPORT_STATUS } from '../import_progress';
@@ -76,14 +77,17 @@ const DEFAULT_STATE = {
combinedFields: [],
importer: undefined,
createPipeline: true,
+ initializeDeployment: false,
+ initializeDeploymentStatus: IMPORT_STATUS.INCOMPLETE,
};
export class ImportView extends Component {
+ static contextType = context;
+
constructor(props) {
super(props);
this.state = getDefaultState(DEFAULT_STATE, this.props.results, this.props.capabilities);
- this.dataViewsContract = props.dataViewsContract;
}
componentDidMount() {
@@ -98,7 +102,12 @@ export class ImportView extends Component {
};
clickImport = () => {
- const { data, results, dataViewsContract, fileUpload } = this.props;
+ const { data, results } = this.props;
+ const {
+ data: { dataViews: dataViewsContract },
+ fileUpload,
+ http,
+ } = this.context.services;
const {
index,
dataView,
@@ -110,12 +119,13 @@ export class ImportView extends Component {
} = this.state;
const createPipeline = pipelineString !== '';
+
this.setState({
createPipeline,
});
importData(
- { data, results, dataViewsContract, fileUpload },
+ { data, results, dataViewsContract, fileUpload, http },
{
index,
dataView,
@@ -150,7 +160,7 @@ export class ImportView extends Component {
return;
}
- const exists = await this.props.fileUpload.checkIndexExists(index);
+ const exists = await this.context.services.fileUpload.checkIndexExists(index);
const indexNameError = exists ? (
)}
diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx
index 01d9d2c37194f..6289a73e2a664 100644
--- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx
+++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx
@@ -44,7 +44,6 @@ export const FileDataVisualizer: FC = ({ getAdditionalLinks, resultLinks
, logger: Logger)
const filteredInferenceEndpoints = endpoints.filter((endpoint) => {
return (
- (endpoint.task_type === 'sparse_embedding' ||
- endpoint.task_type === 'text_embedding') &&
- endpoint.service_settings.num_allocations >= 0
+ endpoint.task_type === 'sparse_embedding' || endpoint.task_type === 'text_embedding'
);
});
@@ -108,4 +106,50 @@ export function routes(coreSetup: CoreSetup, logger: Logger)
}
}
);
+
+ /**
+ * @apiGroup DataVisualizer
+ *
+ * @api {get} /internal/data_visualizer/inference/{inferenceId} Runs inference on a given inference endpoint with the provided input
+ * @apiName inference
+ * @apiDescription Runs inference on a given inference endpoint with the provided input.
+ */
+ router.versioned
+ .post({
+ path: '/internal/data_visualizer/inference/{inferenceId}',
+ access: 'internal',
+ options: {
+ tags: ['access:fileUpload:analyzeFile'],
+ },
+ })
+ .addVersion(
+ {
+ version: '1',
+ validate: {
+ request: {
+ params: schema.object({
+ inferenceId: schema.string(),
+ }),
+ body: schema.object({
+ input: schema.string(),
+ }),
+ },
+ },
+ },
+ async (context, request, response) => {
+ try {
+ const inferenceId = request.params.inferenceId;
+ const input = request.body.input;
+ const esClient = (await context.core).elasticsearch.client;
+ const body = await esClient.asCurrentUser.inference.inference({
+ inference_id: inferenceId,
+ input,
+ });
+
+ return response.ok({ body });
+ } catch (e) {
+ return response.customError(wrapError(e));
+ }
+ }
+ );
}