From c90f721810c1bde3d8fd4bdbee6eaaf7cfe7f9be Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 1 Apr 2024 12:02:39 -0700 Subject: [PATCH 01/11] Fix reactflow context; set up hardcoded flow -> template conversion logic Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + common/interfaces.ts | 2 +- common/utils.ts | 95 +++++++++++++------ .../input_fields/select_field.tsx | 4 +- .../pages/workflow_detail/workflow_detail.tsx | 10 +- .../workspace/reactflow-styles.scss | 2 - .../workspace/resizable_workspace.tsx | 63 +++++++----- 7 files changed, 112 insertions(+), 65 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index eb10828f..e86b9d6c 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -37,5 +37,6 @@ export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH export const NEW_WORKFLOW_ID_URL = 'new'; export const START_FROM_SCRATCH_WORKFLOW_NAME = 'Start From Scratch'; export const DEFAULT_NEW_WORKFLOW_NAME = 'new_workflow'; +export const DEFAULT_NEW_WORKFLOW_DESCRIPTION = 'My new workflow'; export const DATE_FORMAT_PATTERN = 'MM/DD/YY hh:mm A'; export const EMPTY_FIELD_STRING = '--'; diff --git a/common/interfaces.ts b/common/interfaces.ts index b559d47e..353c402f 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -51,7 +51,7 @@ export type TemplateEdge = { }; export type TemplateFlow = { - user_params?: Map; + user_inputs?: Map; nodes: TemplateNode[]; edges?: TemplateEdge[]; }; diff --git a/common/utils.ts b/common/utils.ts index 2e5bf2d9..08a316f4 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -17,21 +17,52 @@ import { DATE_FORMAT_PATTERN, COMPONENT_CATEGORY, NODE_CATEGORY, + WorkspaceFormValues, } from './'; // TODO: implement this and remove hardcoded return values /** - * Converts a ReactFlow workspace flow to a backend-compatible set of ingest and/or search sub-workflows, - * along with a provision sub-workflow if resources are to be created. + * Given a ReactFlow workspace flow and the set of current form values within such flow, + * generate a backend-compatible set of sub-workflows. + * */ export function toTemplateFlows( - workspaceFlow: WorkspaceFlowState + workspaceFlow: WorkspaceFlowState, + formValues: WorkspaceFormValues ): TemplateFlows { + const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) => + key.includes('text_embedding') + ) as string; + const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId]; + return { provision: { - user_params: {} as Map, - nodes: [], - edges: [], + nodes: [ + { + id: 'create_ingest_pipeline', + type: 'create_ingest_pipeline', + user_inputs: { + pipeline_id: 'test-pipeline', + model_id: textEmbeddingFields['modelId'], + input_field: textEmbeddingFields['inputField'], + output_field: textEmbeddingFields['outputField'], + configurations: { + description: 'A text embedding ingest pipeline', + processors: [ + { + text_embedding: { + model_id: '${{user_inputs.model_id}}', + field_map: { + '${{user_inputs.input_field}}': + '${{user_inputs.output_field}}', + }, + }, + }, + ], + }, + }, + }, + ], }, }; } @@ -61,11 +92,10 @@ export function toWorkspaceFlow( style: { width: 900, height: 400, - overflowX: 'auto', - overflowY: 'auto', }, className: 'reactflow__group-node__ingest', selectable: true, + deletable: false, }, { id: ingestId1, @@ -78,6 +108,7 @@ export function toWorkspaceFlow( parentNode: ingestGroupId, extent: 'parent', draggable: true, + deletable: false, }, { id: ingestId2, @@ -87,6 +118,7 @@ export function toWorkspaceFlow( parentNode: ingestGroupId, extent: 'parent', draggable: true, + deletable: false, }, ] as ReactFlowComponent[]; @@ -99,33 +131,34 @@ export function toWorkspaceFlow( style: { width: 900, height: 400, - overflowX: 'auto', - overflowY: 'auto', }, className: 'reactflow__group-node__search', selectable: true, + deletable: false, }, - { - id: searchId1, - position: { x: 100, y: 70 }, - data: initComponentData( - new TextEmbeddingTransformer().toObj(), - searchId1 - ), - type: NODE_CATEGORY.CUSTOM, - parentNode: searchGroupId, - extent: 'parent', - draggable: true, - }, - { - id: searchId2, - position: { x: 500, y: 70 }, - data: initComponentData(new KnnIndexer().toObj(), searchId2), - type: NODE_CATEGORY.CUSTOM, - parentNode: searchGroupId, - extent: 'parent', - draggable: true, - }, + // { + // id: searchId1, + // position: { x: 100, y: 70 }, + // data: initComponentData( + // new TextEmbeddingTransformer().toObj(), + // searchId1 + // ), + // type: NODE_CATEGORY.CUSTOM, + // parentNode: searchGroupId, + // extent: 'parent', + // draggable: true, + // deletable: false, + // }, + // { + // id: searchId2, + // position: { x: 500, y: 70 }, + // data: initComponentData(new KnnIndexer().toObj(), searchId2), + // type: NODE_CATEGORY.CUSTOM, + // parentNode: searchGroupId, + // extent: 'parent', + // draggable: true, + // deletable: false, + // }, ] as ReactFlowComponent[]; return { diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx index f76567a8..b043fb31 100644 --- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx @@ -22,12 +22,12 @@ import { // Need to have a way to determine where to fetch this dynamic data. const existingIndices = [ { - value: 'index-1', + value: 'my-index-1', inputDisplay: my-index-1, disabled: false, }, { - value: 'index-2', + value: 'my-index-2', inputDisplay: my-index-2, disabled: false, }, diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 5907d75d..a59d3cf9 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -144,10 +144,12 @@ export function WorkflowDetail(props: WorkflowDetailProps) { tabs={tabs} /> {selectedTabId === WORKFLOW_DETAILS_TAB.EDITOR && ( - + + + )} {selectedTabId === WORKFLOW_DETAILS_TAB.LAUNCHES && } {selectedTabId === WORKFLOW_DETAILS_TAB.PROTOTYPE && ( diff --git a/public/pages/workflow_detail/workspace/reactflow-styles.scss b/public/pages/workflow_detail/workspace/reactflow-styles.scss index 900bdc45..c4fc599d 100644 --- a/public/pages/workflow_detail/workspace/reactflow-styles.scss +++ b/public/pages/workflow_detail/workspace/reactflow-styles.scss @@ -25,8 +25,6 @@ $handle-color-invalid: $euiColorDanger; .reactflow__group-node { width: 1200px; height: 700px; - overflow-x: auto; - overflow-y: auto; border: 'none'; &__ingest { diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index 11f93956..e6f14185 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -5,7 +5,7 @@ import React, { useRef, useState, useEffect } from 'react'; import { useDispatch, useSelector } from 'react-redux'; -import { ReactFlowProvider, useReactFlow } from 'reactflow'; +import { useReactFlow } from 'reactflow'; import { Form, Formik } from 'formik'; import * as yup from 'yup'; import { cloneDeep } from 'lodash'; @@ -29,6 +29,9 @@ import { validateWorkspaceFlow, WorkspaceFlowState, toTemplateFlows, + DEFAULT_NEW_WORKFLOW_NAME, + DEFAULT_NEW_WORKFLOW_DESCRIPTION, + USE_CASE, } from '../../../../common'; import { AppState, removeDirty, setDirty } from '../../../store'; import { Workspace } from './workspace'; @@ -107,22 +110,32 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { } } - // Hook to update the workflow's flow state, if applicable. It may not exist if - // it is a backend-only-created workflow, or a new, unsaved workflow. If so, - // generate a default one based on the 'workflows' JSON field. + // Hook to update some default values for the workflow, if applicable. Flow state + // may not exist if it is a backend-only-created workflow, or a new, unsaved workflow. + // Metadata fields (name/description/use_case/etc.) may not exist if the user + // cold reloads the page on a new, unsaved workflow. useEffect(() => { - const workflowCopy = { ...props.workflow } as Workflow; - if (workflowCopy) { - if (!workflowCopy.workspaceFlowState) { - workflowCopy.workspaceFlowState = toWorkspaceFlow( - workflowCopy.workflows - ); - console.debug( - `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` - ); - } - setWorkflow(workflowCopy); + let workflowCopy = { ...props.workflow } as Workflow; + if (!workflowCopy.workspaceFlowState) { + workflowCopy.workspaceFlowState = toWorkspaceFlow(workflowCopy.workflows); + console.debug( + `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` + ); } + + // TODO: tune some of the defaults, like use_case and version as these will change + workflowCopy = { + ...workflowCopy, + name: workflowCopy.name || DEFAULT_NEW_WORKFLOW_NAME, + description: workflowCopy.description || DEFAULT_NEW_WORKFLOW_DESCRIPTION, + use_case: workflowCopy.use_case || USE_CASE.PROVISION, + version: workflowCopy.version || { + template: '1.0.0', + compatibility: ['2.12.0', '3.0.0'], + }, + }; + + setWorkflow(workflowCopy); }, [props.workflow]); // Hook to updated the selected ReactFlow component @@ -250,7 +263,6 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { setFormValidOnSubmit(false); } else { setFormValidOnSubmit(true); - // @ts-ignore let curFlowState = reactFlowInstance.toObject() as WorkspaceFlowState; curFlowState = { ...curFlowState, @@ -261,7 +273,10 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { const updatedWorkflow = { ...workflow, workspaceFlowState: curFlowState, - workflows: toTemplateFlows(curFlowState), + workflows: toTemplateFlows( + curFlowState, + formikProps.values + ), } as Workflow; saveWorkflow(updatedWorkflow); } else { @@ -302,14 +317,12 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { className="workspace-panel" > - - - + From 2bb403a2f0892c08be143805505cd44db11126f0 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 1 Apr 2024 14:48:24 -0700 Subject: [PATCH 02/11] Switch inputs to createFields; clean up sample template Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 1 + common/utils.ts | 61 +++++++++++-------- public/component_types/indexer/indexer.ts | 29 ++++----- public/component_types/indexer/knn_indexer.ts | 16 ++--- .../transformer/text_embedding_transformer.ts | 2 +- .../component_details/component_inputs.tsx | 32 ++++++++-- .../component_details/input_field_list.tsx | 12 ++-- .../new_or_existing_tabs.tsx | 10 +-- .../workspace_edge/deletable-edge-styles.scss | 6 +- .../workspace_edge/deletable_edge.tsx | 3 +- public/utils/utils.ts | 8 ++- 11 files changed, 110 insertions(+), 70 deletions(-) diff --git a/common/interfaces.ts b/common/interfaces.ts index 353c402f..d8b15a09 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -52,6 +52,7 @@ export type TemplateEdge = { export type TemplateFlow = { user_inputs?: Map; + previous_node_inputs?: Map; nodes: TemplateNode[]; edges?: TemplateEdge[]; }; diff --git a/common/utils.ts b/common/utils.ts index 08a316f4..78cde047 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -62,6 +62,41 @@ export function toTemplateFlows( }, }, }, + { + id: 'create_index', + type: 'create_index', + previous_node_inputs: { + create_ingest_pipeline: 'pipeline_id', + }, + user_inputs: { + index_name: '${{create_index.name}}', + configurations: { + settings: { + default_pipeline: '${{create_ingest_pipeline.pipeline_id}}', + }, + mappings: { + properties: { + id: { + type: 'text', + }, + [textEmbeddingFields['outputField']]: { + type: 'knn_vector', + dimension: 768, + method: { + engine: 'lucene', + space_type: 'l2', + name: 'hnsw', + parameters: {}, + }, + }, + [textEmbeddingFields['inputField']]: { + type: 'text', + }, + }, + }, + }, + }, + }, ], }, }; @@ -78,9 +113,6 @@ export function toWorkspaceFlow( const ingestId1 = generateId('text_embedding_processor'); const ingestId2 = generateId('knn_index'); const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); - - const searchId1 = generateId('text_embedding_processor'); - const searchId2 = generateId('knn_index'); const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); const ingestNodes = [ @@ -136,29 +168,6 @@ export function toWorkspaceFlow( selectable: true, deletable: false, }, - // { - // id: searchId1, - // position: { x: 100, y: 70 }, - // data: initComponentData( - // new TextEmbeddingTransformer().toObj(), - // searchId1 - // ), - // type: NODE_CATEGORY.CUSTOM, - // parentNode: searchGroupId, - // extent: 'parent', - // draggable: true, - // deletable: false, - // }, - // { - // id: searchId2, - // position: { x: 500, y: 70 }, - // data: initComponentData(new KnnIndexer().toObj(), searchId2), - // type: NODE_CATEGORY.CUSTOM, - // parentNode: searchGroupId, - // extent: 'parent', - // draggable: true, - // deletable: false, - // }, ] as ReactFlowComponent[]; return { diff --git a/public/component_types/indexer/indexer.ts b/public/component_types/indexer/indexer.ts index e87a81b0..eabf2360 100644 --- a/public/component_types/indexer/indexer.ts +++ b/public/component_types/indexer/indexer.ts @@ -46,20 +46,21 @@ export class Indexer extends BaseComponent { optional: false, advanced: false, }, - { - label: 'Mappings', - name: 'indexMappings', - type: 'json', - placeholder: 'Enter an index mappings JSON blob...', - optional: false, - advanced: false, - }, - ]; - this.outputs = [ - { - label: this.label, - baseClasses: this.baseClasses, - }, + // { + // label: 'Mappings', + // name: 'indexMappings', + // type: 'json', + // placeholder: 'Enter an index mappings JSON blob...', + // optional: false, + // advanced: false, + // }, ]; + // this.outputs = [ + // { + // label: this.label, + // baseClasses: this.baseClasses, + // }, + // ]; + this.outputs = []; } } diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts index 93f55017..77647dfa 100644 --- a/public/component_types/indexer/knn_indexer.ts +++ b/public/component_types/indexer/knn_indexer.ts @@ -17,14 +17,14 @@ export class KnnIndexer extends Indexer { // @ts-ignore ...this.createFields, // TODO: finalize what to expose / what to have for defaults here - { - label: 'K-NN Settings', - name: 'knnSettings', - type: 'json', - placeholder: 'Enter K-NN settings JSON blob...', - optional: false, - advanced: false, - }, + // { + // label: 'K-NN Settings', + // name: 'knnSettings', + // type: 'json', + // placeholder: 'Enter K-NN settings JSON blob...', + // optional: false, + // advanced: false, + // }, ]; } } diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index dbdce094..53164ca8 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -14,7 +14,7 @@ export class TextEmbeddingTransformer extends MLTransformer { this.label = 'Text Embedding Transformer'; this.description = 'A specialized ML transformer for embedding text'; this.inputs = []; - this.fields = [ + this.createFields = [ { label: 'Model ID', name: 'modelId', diff --git a/public/pages/workflow_detail/component_details/component_inputs.tsx b/public/pages/workflow_detail/component_details/component_inputs.tsx index 3d9f076e..5f49895b 100644 --- a/public/pages/workflow_detail/component_details/component_inputs.tsx +++ b/public/pages/workflow_detail/component_details/component_inputs.tsx @@ -3,10 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useState } from 'react'; import { EuiSpacer, EuiText, EuiTitle } from '@elastic/eui'; import { InputFieldList } from './input_field_list'; import { NODE_CATEGORY, ReactFlowComponent } from '../../../../common'; +import { NewOrExistingTabs } from '../workspace/workspace_components/new_or_existing_tabs'; interface ComponentInputsProps { selectedComponent: ReactFlowComponent; @@ -14,6 +15,13 @@ interface ComponentInputsProps { } export function ComponentInputs(props: ComponentInputsProps) { + // Tab state + enum TAB { + NEW = 'new', + EXISTING = 'existing', + } + const [selectedTabId, setSelectedTabId] = useState(TAB.NEW); + // Have custom layouts for parent/group flows if (props.selectedComponent.type === NODE_CATEGORY.INGEST_GROUP) { return ( @@ -47,11 +55,25 @@ export function ComponentInputs(props: ComponentInputsProps) {

{props.selectedComponent.data.label || ''}

 - - + + {selectedTabId === TAB.NEW && ( + + )} + {selectedTabId === TAB.EXISTING && ( + + )} ); } diff --git a/public/pages/workflow_detail/component_details/input_field_list.tsx b/public/pages/workflow_detail/component_details/input_field_list.tsx index f7aed9d8..843ba8ea 100644 --- a/public/pages/workflow_detail/component_details/input_field_list.tsx +++ b/public/pages/workflow_detail/component_details/input_field_list.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { TextField, JsonField, SelectField } from './input_fields'; -import { ReactFlowComponent } from '../../../../common'; +import { IComponentField } from '../../../../common'; /** * A helper component to format all of the input fields for a component. Dynamically @@ -14,12 +14,13 @@ import { ReactFlowComponent } from '../../../../common'; */ interface InputFieldListProps { - selectedComponent: ReactFlowComponent; + componentId: string; + componentFields: IComponentField[] | undefined; onFormChange: () => void; } export function InputFieldList(props: InputFieldListProps) { - const inputFields = props.selectedComponent.data.fields || []; + const inputFields = props.componentFields || []; return ( {inputFields.map((field, idx) => { @@ -30,7 +31,7 @@ export function InputFieldList(props: InputFieldListProps) { @@ -43,9 +44,10 @@ export function InputFieldList(props: InputFieldListProps) { + ); break; diff --git a/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx b/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx index 3bc7d1e2..f5a936fc 100644 --- a/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx +++ b/public/pages/workflow_detail/workspace/workspace_components/new_or_existing_tabs.tsx @@ -16,16 +16,16 @@ interface NewOrExistingTabsProps { } const inputTabs = [ - { - id: 'existing', - name: 'Existing', - disabled: false, - }, { id: 'new', name: 'New', disabled: false, }, + { + id: 'existing', + name: 'Existing', + disabled: true, + }, ]; export function NewOrExistingTabs(props: NewOrExistingTabsProps) { diff --git a/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss b/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss index a249a641..62ed2776 100644 --- a/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss +++ b/public/pages/workflow_detail/workspace/workspace_edge/deletable-edge-styles.scss @@ -1,8 +1,8 @@ .delete-edge-button { width: 20px; height: 20px; - background: #eee; - border: 1px solid #fff; + background: #000000; + border: 1px solid #000000; cursor: pointer; border-radius: 50%; font-size: 12px; @@ -10,5 +10,5 @@ } .delete-edge-button:hover { - box-shadow: 0 0 6px 2px rgba(0, 0, 0, 0.08); + box-shadow: 0 0 6px 2px rgba(0, 0, 0, 0.5); } diff --git a/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx b/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx index 81ca6063..d9c9ff4d 100644 --- a/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx +++ b/public/pages/workflow_detail/workspace/workspace_edge/deletable_edge.tsx @@ -9,6 +9,7 @@ import { Edge, EdgeLabelRenderer, EdgeProps, + MarkerType, getBezierPath, useReactFlow, } from 'reactflow'; @@ -64,7 +65,7 @@ export function DeletableEdge(props: DeletableEdgeProps) { transform: `translate(-50%, -50%) translate(${labelX}px,${labelY}px)`, fontSize: 12, pointerEvents: 'all', - zIndex: 1, + zIndex: 3, }} className="nodrag nopan" > diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 5eaa77c1..8df05bc7 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -42,10 +42,12 @@ export function initComponentData( **************** Formik (form) utils ********************** */ +// TODO: below, we are hardcoding to only persisting and validating create fields. +// If we support both, we will need to dynamically update. // Converting stored values in component data to initial formik values export function componentDataToFormik(data: IComponentData): FormikValues { const formikValues = {} as FormikValues; - data.fields?.forEach((field) => { + data.createFields?.forEach((field) => { formikValues[field.name] = field.value || getInitialValue(field.type); }); return formikValues; @@ -99,9 +101,11 @@ export function getFieldError( **************** Yup (validation) utils ********************** */ +// TODO: below, we are hardcoding to only persisting and validating create fields. +// If we support both, we will need to dynamically update. export function getComponentSchema(data: IComponentData): ObjectSchema { const schemaObj = {} as { [key: string]: Schema }; - data.fields?.forEach((field) => { + data.createFields?.forEach((field) => { schemaObj[field.name] = getFieldSchema(field); }); return yup.object(schemaObj); From ad260eee288adddb3c1b305c74fbf8b611eb3f23 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 1 Apr 2024 15:07:42 -0700 Subject: [PATCH 03/11] Remove custom edge for now Signed-off-by: Tyler Ohlsen --- common/utils.ts | 15 ++++++++++++++- .../workflow_detail/workspace/workspace.tsx | 18 ++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/common/utils.ts b/common/utils.ts index 78cde047..dd424475 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -4,6 +4,7 @@ */ import moment from 'moment'; +import { MarkerType } from 'reactflow'; import { WorkspaceFlowState, ReactFlowComponent, @@ -172,7 +173,19 @@ export function toWorkspaceFlow( return { nodes: [...ingestNodes, ...searchNodes], - edges: [] as ReactFlowEdge[], + edges: [ + { + source: ingestId1, + target: ingestId2, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + zIndex: 2, + deletable: false, + }, + ] as ReactFlowEdge[], }; } diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index 72162e66..4303378f 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -15,6 +15,7 @@ import ReactFlow, { useStore, useReactFlow, useOnSelectionChange, + MarkerType, } from 'reactflow'; import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { setDirty } from '../../../store'; @@ -82,7 +83,19 @@ export function Workspace(props: WorkspaceProps) { ...params, type: 'customEdge', }; - setEdges((eds) => addEdge(edge, eds)); + setEdges((eds) => + addEdge( + { + ...edge, + markerEnd: { + type: MarkerType.ArrowClosed, + width: 20, + height: 20, + }, + }, + eds + ) + ); dispatch(setDirty()); }, [setEdges] @@ -115,7 +128,8 @@ export function Workspace(props: WorkspaceProps) { nodes={nodes} edges={edges} nodeTypes={nodeTypes} - edgeTypes={edgeTypes} + // TODO: add custom edge types back if we want to support custom deletable buttons + // edgeTypes={edgeTypes} onNodesChange={onNodesChange} onEdgesChange={onEdgesChange} onConnect={onConnect} From 691695ad2a5faf851ab396171601b6a3d398dbe9 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 1 Apr 2024 17:05:31 -0700 Subject: [PATCH 04/11] Refactor into uiMetadata; support for workflow creation Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 8 +++-- common/utils.ts | 9 +++++- public/pages/workflow_detail/utils/utils.ts | 10 +----- .../workspace/resizable_workspace.tsx | 32 ++++++++++++++----- .../workflow_detail/workspace/workspace.tsx | 6 ++-- public/store/reducers/workflows_reducer.ts | 17 ++-------- 6 files changed, 45 insertions(+), 37 deletions(-) diff --git a/common/interfaces.ts b/common/interfaces.ts index d8b15a09..f2d85fc0 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -28,6 +28,10 @@ type ReactFlowViewport = { zoom: number; }; +export type UIState = { + workspaceFlow: WorkspaceFlowState; +}; + export type WorkspaceFlowState = { nodes: ReactFlowComponent[]; edges: ReactFlowEdge[]; @@ -76,8 +80,8 @@ export type WorkflowTemplate = { export type Workflow = WorkflowTemplate & { // won't exist until created in backend id?: string; - // ReactFlow state may not exist if a workflow is created via API/backend-only. - workspaceFlowState?: WorkspaceFlowState; + // UI state and any ReactFlow state may not exist if a workflow is created via API/backend-only. + uiMetadata?: UIState; // won't exist until created in backend lastUpdated?: number; // won't exist until launched/provisioned in backend diff --git a/common/utils.ts b/common/utils.ts index dd424475..89b5a91b 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -34,7 +34,11 @@ export function toTemplateFlows( const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) => key.includes('text_embedding') ) as string; + const knnIndexerNodeId = Object.keys(formValues).find((key) => + key.includes('knn') + ) as string; const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId]; + const knnIndexerFields = formValues[knnIndexerNodeId]; return { provision: { @@ -70,7 +74,7 @@ export function toTemplateFlows( create_ingest_pipeline: 'pipeline_id', }, user_inputs: { - index_name: '${{create_index.name}}', + index_name: knnIndexerFields['indexName'], configurations: { settings: { default_pipeline: '${{create_ingest_pipeline.pipeline_id}}', @@ -115,6 +119,7 @@ export function toWorkspaceFlow( const ingestId2 = generateId('knn_index'); const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST); const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH); + const edgeId = generateId('edge'); const ingestNodes = [ { @@ -175,6 +180,8 @@ export function toWorkspaceFlow( nodes: [...ingestNodes, ...searchNodes], edges: [ { + id: edgeId, + key: edgeId, source: ingestId1, target: ingestId2, markerEnd: { diff --git a/public/pages/workflow_detail/utils/utils.ts b/public/pages/workflow_detail/utils/utils.ts index b3037a1c..5108c519 100644 --- a/public/pages/workflow_detail/utils/utils.ts +++ b/public/pages/workflow_detail/utils/utils.ts @@ -3,15 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Workflow, ReactFlowComponent } from '../../../../common'; - -export function saveWorkflow(workflow?: Workflow): void { - if (workflow && workflow.id) { - // TODO: implement connection to update workflow API - } else { - // TODO: implement connection to create workflow API - } -} +import { ReactFlowComponent } from '../../../../common'; // Process the raw ReactFlow nodes to only persist the fields we need export function processNodes( diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index e6f14185..aaccbb12 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -33,10 +33,15 @@ import { DEFAULT_NEW_WORKFLOW_DESCRIPTION, USE_CASE, } from '../../../../common'; -import { AppState, removeDirty, setDirty } from '../../../store'; +import { + AppState, + createWorkflow, + removeDirty, + setDirty, +} from '../../../store'; import { Workspace } from './workspace'; import { ComponentDetails } from '../component_details'; -import { processNodes, saveWorkflow } from '../utils'; +import { processNodes } from '../utils'; // styling import './workspace-styles.scss'; @@ -116,8 +121,11 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // cold reloads the page on a new, unsaved workflow. useEffect(() => { let workflowCopy = { ...props.workflow } as Workflow; - if (!workflowCopy.workspaceFlowState) { - workflowCopy.workspaceFlowState = toWorkspaceFlow(workflowCopy.workflows); + if (!workflowCopy.uiMetadata || !workflowCopy.uiMetadata.workspaceFlow) { + workflowCopy.uiMetadata = { + ...(workflowCopy.uiMetadata || {}), + workspaceFlow: toWorkspaceFlow(workflowCopy.workflows), + }; console.debug( `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.` ); @@ -153,10 +161,10 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // Initialize the form state to an existing workflow, if applicable. useEffect(() => { - if (workflow?.workspaceFlowState) { + if (workflow?.uiMetadata?.workspaceFlow) { const initFormValues = {} as WorkspaceFormValues; const initSchemaObj = {} as WorkspaceSchemaObj; - workflow.workspaceFlowState.nodes.forEach((node) => { + workflow.uiMetadata.workspaceFlow.nodes.forEach((node) => { initFormValues[node.id] = componentDataToFormik(node.data); initSchemaObj[node.id] = getComponentSchema(node.data); }); @@ -272,13 +280,21 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { setFlowValidOnSubmit(true); const updatedWorkflow = { ...workflow, - workspaceFlowState: curFlowState, + uiMetadata: { + ...workflow?.uiMetadata, + workspaceFlow: curFlowState, + }, workflows: toTemplateFlows( curFlowState, formikProps.values ), } as Workflow; - saveWorkflow(updatedWorkflow); + if (updatedWorkflow.id) { + // TODO: add update workflow API + } else { + console.log('creating workflow: ', workflow); + dispatch(createWorkflow(updatedWorkflow)); + } } else { setFlowValidOnSubmit(false); } diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index 4303378f..6260c9f4 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -104,9 +104,9 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Set the nodes and edges to an existing workflow state, useEffect(() => { const workflow = { ...props.workflow }; - if (workflow && workflow.workspaceFlowState) { - setNodes(workflow.workspaceFlowState.nodes); - setEdges(workflow.workspaceFlowState.edges); + if (workflow?.uiMetadata?.workspaceFlow) { + setNodes(workflow.uiMetadata.workspaceFlow.nodes); + setEdges(workflow.uiMetadata.workspaceFlow.edges); } }, [props.workflow]); diff --git a/public/store/reducers/workflows_reducer.ts b/public/store/reducers/workflows_reducer.ts index 1092d605..bc0ae203 100644 --- a/public/store/reducers/workflows_reducer.ts +++ b/public/store/reducers/workflows_reducer.ts @@ -4,18 +4,7 @@ */ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; -import { - Workflow, - ReactFlowComponent, - ReactFlowEdge, - KnnIndexer, - TextEmbeddingTransformer, - generateId, - initComponentData, - WORKFLOW_STATE, - WorkflowDict, - WorkflowTemplate, -} from '../../../common'; +import { Workflow, WorkflowDict } from '../../../common'; import { HttpFetchError } from '../../../../../src/core/public'; import { getRouteService } from '../../services'; @@ -85,10 +74,10 @@ export const getWorkflowState = createAsyncThunk( export const createWorkflow = createAsyncThunk( CREATE_WORKFLOW_ACTION, - async (body: {}, { rejectWithValue }) => { + async (workflowBody: {}, { rejectWithValue }) => { const response: | any - | HttpFetchError = await getRouteService().createWorkflow(body); + | HttpFetchError = await getRouteService().createWorkflow(workflowBody); if (response instanceof HttpFetchError) { return rejectWithValue( 'Error creating workflow: ' + response.body.message From e7e30b7996c9d9560988fe32b6e9a5c8374b0acd Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Mon, 1 Apr 2024 17:58:46 -0700 Subject: [PATCH 05/11] Support optional provision in createWorkflow; update ui_metadata field Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 4 ++-- .../workspace/resizable_workspace.tsx | 18 ++++++++++-------- .../workflow_detail/workspace/workspace.tsx | 6 +++--- public/route_service.ts | 9 ++++++--- public/store/reducers/workflows_reducer.ts | 5 ++++- server/cluster/flow_framework_plugin.ts | 8 +++++++- server/routes/flow_framework_routes_service.ts | 9 ++++++--- 7 files changed, 38 insertions(+), 21 deletions(-) diff --git a/common/interfaces.ts b/common/interfaces.ts index f2d85fc0..2f3f0a74 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -74,14 +74,14 @@ export type WorkflowTemplate = { // https://github.com/opensearch-project/flow-framework/issues/526 version: any; workflows: TemplateFlows; + // UI state and any ReactFlow state may not exist if a workflow is created via API/backend-only. + ui_metadata?: UIState; }; // An instance of a workflow based on a workflow template export type Workflow = WorkflowTemplate & { // won't exist until created in backend id?: string; - // UI state and any ReactFlow state may not exist if a workflow is created via API/backend-only. - uiMetadata?: UIState; // won't exist until created in backend lastUpdated?: number; // won't exist until launched/provisioned in backend diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index aaccbb12..df882902 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -121,9 +121,9 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // cold reloads the page on a new, unsaved workflow. useEffect(() => { let workflowCopy = { ...props.workflow } as Workflow; - if (!workflowCopy.uiMetadata || !workflowCopy.uiMetadata.workspaceFlow) { - workflowCopy.uiMetadata = { - ...(workflowCopy.uiMetadata || {}), + if (!workflowCopy.ui_metadata || !workflowCopy.ui_metadata.workspaceFlow) { + workflowCopy.ui_metadata = { + ...(workflowCopy.ui_metadata || {}), workspaceFlow: toWorkspaceFlow(workflowCopy.workflows), }; console.debug( @@ -161,10 +161,10 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { // Initialize the form state to an existing workflow, if applicable. useEffect(() => { - if (workflow?.uiMetadata?.workspaceFlow) { + if (workflow?.ui_metadata?.workspaceFlow) { const initFormValues = {} as WorkspaceFormValues; const initSchemaObj = {} as WorkspaceSchemaObj; - workflow.uiMetadata.workspaceFlow.nodes.forEach((node) => { + workflow.ui_metadata.workspaceFlow.nodes.forEach((node) => { initFormValues[node.id] = componentDataToFormik(node.data); initSchemaObj[node.id] = getComponentSchema(node.data); }); @@ -280,8 +280,8 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { setFlowValidOnSubmit(true); const updatedWorkflow = { ...workflow, - uiMetadata: { - ...workflow?.uiMetadata, + ui_metadata: { + ...workflow?.ui_metadata, workspaceFlow: curFlowState, }, workflows: toTemplateFlows( @@ -292,7 +292,9 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { if (updatedWorkflow.id) { // TODO: add update workflow API } else { - console.log('creating workflow: ', workflow); + // TODO: keep ui_metadata field after backend data model is fixed + // and supports it + delete updatedWorkflow.ui_metadata; dispatch(createWorkflow(updatedWorkflow)); } } else { diff --git a/public/pages/workflow_detail/workspace/workspace.tsx b/public/pages/workflow_detail/workspace/workspace.tsx index 6260c9f4..b5af1b0c 100644 --- a/public/pages/workflow_detail/workspace/workspace.tsx +++ b/public/pages/workflow_detail/workspace/workspace.tsx @@ -104,9 +104,9 @@ export function Workspace(props: WorkspaceProps) { // Initialization. Set the nodes and edges to an existing workflow state, useEffect(() => { const workflow = { ...props.workflow }; - if (workflow?.uiMetadata?.workspaceFlow) { - setNodes(workflow.uiMetadata.workspaceFlow.nodes); - setEdges(workflow.uiMetadata.workspaceFlow.edges); + if (workflow?.ui_metadata?.workspaceFlow) { + setNodes(workflow.ui_metadata.workspaceFlow.nodes); + setEdges(workflow.ui_metadata.workspaceFlow.edges); } }, [props.workflow]); diff --git a/public/route_service.ts b/public/route_service.ts index 6130e2a6..334d0ddb 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -25,7 +25,10 @@ export interface RouteService { getWorkflow: (workflowId: string) => Promise; searchWorkflows: (body: {}) => Promise; getWorkflowState: (workflowId: string) => Promise; - createWorkflow: (body: {}) => Promise; + createWorkflow: ( + body: {}, + provision?: boolean + ) => Promise; deleteWorkflow: (workflowId: string) => Promise; getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; @@ -66,10 +69,10 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, - createWorkflow: async (body: {}) => { + createWorkflow: async (body: {}, provision: boolean = false) => { try { const response = await core.http.post<{ respString: string }>( - CREATE_WORKFLOW_NODE_API_PATH, + `${CREATE_WORKFLOW_NODE_API_PATH}/${provision}`, { body: JSON.stringify(body), } diff --git a/public/store/reducers/workflows_reducer.ts b/public/store/reducers/workflows_reducer.ts index bc0ae203..f748f08b 100644 --- a/public/store/reducers/workflows_reducer.ts +++ b/public/store/reducers/workflows_reducer.ts @@ -77,7 +77,10 @@ export const createWorkflow = createAsyncThunk( async (workflowBody: {}, { rejectWithValue }) => { const response: | any - | HttpFetchError = await getRouteService().createWorkflow(workflowBody); + | HttpFetchError = await getRouteService().createWorkflow( + workflowBody, + false + ); if (response instanceof HttpFetchError) { return rejectWithValue( 'Error creating workflow: ' + response.body.message diff --git a/server/cluster/flow_framework_plugin.ts b/server/cluster/flow_framework_plugin.ts index 3b7616dc..2e649a4b 100644 --- a/server/cluster/flow_framework_plugin.ts +++ b/server/cluster/flow_framework_plugin.ts @@ -67,7 +67,13 @@ export function flowFrameworkPlugin(Client: any, config: any, components: any) { flowFramework.createWorkflow = ca({ url: { - fmt: FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX, + fmt: `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}?provision=<%=provision%>`, + req: { + provision: { + type: 'boolean', + required: true, + }, + }, }, needBody: true, method: 'POST', diff --git a/server/routes/flow_framework_routes_service.ts b/server/routes/flow_framework_routes_service.ts index 19c802a5..4eda47b5 100644 --- a/server/routes/flow_framework_routes_service.ts +++ b/server/routes/flow_framework_routes_service.ts @@ -69,9 +69,12 @@ export function registerFlowFrameworkRoutes( router.post( { - path: CREATE_WORKFLOW_NODE_API_PATH, + path: `${CREATE_WORKFLOW_NODE_API_PATH}/{provision}`, validate: { body: schema.any(), + params: schema.object({ + provision: schema.boolean(), + }), }, }, flowFrameworkRoutesService.createWorkflow @@ -180,11 +183,11 @@ export class FlowFrameworkRoutesService { res: OpenSearchDashboardsResponseFactory ): Promise> => { const body = req.body; - + const { provision } = req.params as { provision: boolean }; try { const response = await this.client .asScoped(req) - .callAsCurrentUser('flowFramework.createWorkflow', { body }); + .callAsCurrentUser('flowFramework.createWorkflow', { body, provision }); return res.ok({ body: { id: response._id } }); } catch (err: any) { return generateCustomError(res, err); From 1bcfa43cac848db1e8e67fa728c71bc9d4c74425 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 2 Apr 2024 10:50:32 -0700 Subject: [PATCH 06/11] Get substitution working; re-add ui_metadata with backend fix Signed-off-by: Tyler Ohlsen --- common/utils.ts | 9 +++------ .../workflow_detail/workspace/resizable_workspace.tsx | 3 --- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/common/utils.ts b/common/utils.ts index 89b5a91b..81a7cafb 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -56,10 +56,10 @@ export function toTemplateFlows( processors: [ { text_embedding: { - model_id: '${{user_inputs.model_id}}', + model_id: textEmbeddingFields['modelId'], field_map: { - '${{user_inputs.input_field}}': - '${{user_inputs.output_field}}', + [textEmbeddingFields['inputField']]: + textEmbeddingFields['outputField'], }, }, }, @@ -81,9 +81,6 @@ export function toTemplateFlows( }, mappings: { properties: { - id: { - type: 'text', - }, [textEmbeddingFields['outputField']]: { type: 'knn_vector', dimension: 768, diff --git a/public/pages/workflow_detail/workspace/resizable_workspace.tsx b/public/pages/workflow_detail/workspace/resizable_workspace.tsx index df882902..b1168228 100644 --- a/public/pages/workflow_detail/workspace/resizable_workspace.tsx +++ b/public/pages/workflow_detail/workspace/resizable_workspace.tsx @@ -292,9 +292,6 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) { if (updatedWorkflow.id) { // TODO: add update workflow API } else { - // TODO: keep ui_metadata field after backend data model is fixed - // and supports it - delete updatedWorkflow.ui_metadata; dispatch(createWorkflow(updatedWorkflow)); } } else { From 14338398938906bab35bd0d19656622288d058bd Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 2 Apr 2024 11:50:34 -0700 Subject: [PATCH 07/11] Fix overflow styles Signed-off-by: Tyler Ohlsen --- public/pages/workflow_detail/workflow_detail.tsx | 2 +- public/pages/workflow_detail/workspace/workspace-styles.scss | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index a59d3cf9..527ce0eb 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -137,7 +137,7 @@ export function WorkflowDetail(props: WorkflowDetailProps) { return ( - + Date: Tue, 2 Apr 2024 14:05:16 -0700 Subject: [PATCH 08/11] Improve IComponentField schema; support hint and links on inputs Signed-off-by: Tyler Ohlsen --- common/utils.ts | 6 +-- public/component_types/indexer/indexer.ts | 7 ---- public/component_types/indexer/knn_indexer.ts | 2 - public/component_types/interfaces.ts | 11 +++-- .../transformer/text_embedding_transformer.ts | 25 +++++++---- .../component_details/input_field_list.tsx | 5 ++- .../input_fields/select_field.tsx | 42 +++++++++++-------- .../input_fields/text_field.tsx | 12 +++++- public/utils/utils.ts | 10 +++-- 9 files changed, 69 insertions(+), 51 deletions(-) diff --git a/common/utils.ts b/common/utils.ts index 81a7cafb..0a01b684 100644 --- a/common/utils.ts +++ b/common/utils.ts @@ -50,7 +50,7 @@ export function toTemplateFlows( pipeline_id: 'test-pipeline', model_id: textEmbeddingFields['modelId'], input_field: textEmbeddingFields['inputField'], - output_field: textEmbeddingFields['outputField'], + output_field: textEmbeddingFields['vectorField'], configurations: { description: 'A text embedding ingest pipeline', processors: [ @@ -59,7 +59,7 @@ export function toTemplateFlows( model_id: textEmbeddingFields['modelId'], field_map: { [textEmbeddingFields['inputField']]: - textEmbeddingFields['outputField'], + textEmbeddingFields['vectorField'], }, }, }, @@ -81,7 +81,7 @@ export function toTemplateFlows( }, mappings: { properties: { - [textEmbeddingFields['outputField']]: { + [textEmbeddingFields['vectorField']]: { type: 'knn_vector', dimension: 768, method: { diff --git a/public/component_types/indexer/indexer.ts b/public/component_types/indexer/indexer.ts index eabf2360..781b1f27 100644 --- a/public/component_types/indexer/indexer.ts +++ b/public/component_types/indexer/indexer.ts @@ -25,7 +25,6 @@ export class Indexer extends BaseComponent { // TODO: may need to change to be looser. it should be able to take // in other component types baseClass: COMPONENT_CLASS.TRANSFORMER, - optional: false, acceptMultiple: false, }, ]; @@ -34,8 +33,6 @@ export class Indexer extends BaseComponent { label: 'Index Name', name: 'indexName', type: 'select', - optional: false, - advanced: false, }, ]; this.createFields = [ @@ -43,16 +40,12 @@ export class Indexer extends BaseComponent { label: 'Index Name', name: 'indexName', type: 'string', - optional: false, - advanced: false, }, // { // label: 'Mappings', // name: 'indexMappings', // type: 'json', // placeholder: 'Enter an index mappings JSON blob...', - // optional: false, - // advanced: false, // }, ]; // this.outputs = [ diff --git a/public/component_types/indexer/knn_indexer.ts b/public/component_types/indexer/knn_indexer.ts index 77647dfa..65023604 100644 --- a/public/component_types/indexer/knn_indexer.ts +++ b/public/component_types/indexer/knn_indexer.ts @@ -22,8 +22,6 @@ export class KnnIndexer extends Indexer { // name: 'knnSettings', // type: 'json', // placeholder: 'Enter K-NN settings JSON blob...', - // optional: false, - // advanced: false, // }, ]; } diff --git a/public/component_types/interfaces.ts b/public/component_types/interfaces.ts index e9349cd3..51eda097 100644 --- a/public/component_types/interfaces.ts +++ b/public/component_types/interfaces.ts @@ -25,20 +25,18 @@ export type WorkspaceSchema = ObjectSchema; /** * Represents a single base class as an input handle for a component. - * It may be optional. It may also accept multiples of that class. + * It may accept multiples of that class. */ export interface IComponentInput { id: string; label: string; baseClass: COMPONENT_CLASS; - optional: boolean; acceptMultiple: boolean; } /** * An input field for a component. Specifies enough configuration for the - * UI node to render it properly within the component (show it as optional, - * put it in advanced settings, placeholder values, etc.) + * UI node to render it properly (help text, links, etc.) */ export interface IComponentField { label: string; @@ -46,8 +44,9 @@ export interface IComponentField { name: string; value?: FieldValue; placeholder?: string; - optional?: boolean; - advanced?: boolean; + helpText?: string; + helpLink?: string; + selectOptions?: string[]; } /** diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index 53164ca8..f679ead3 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -18,23 +18,30 @@ export class TextEmbeddingTransformer extends MLTransformer { { label: 'Model ID', name: 'modelId', - type: 'string', - optional: false, - advanced: false, + type: 'select', + selectOptions: ['model-1', 'test-model', 'model-2'], + helpText: 'The deployed text embedding model to use for embedding.', + helpLink: + 'https://opensearch.org/docs/latest/ml-commons-plugin/integrating-ml-models/#choosing-a-model', }, { label: 'Input Field', name: 'inputField', type: 'string', - optional: false, - advanced: false, + helpText: + 'The name of the field from which to obtain text for generating text embeddings.', + helpLink: + 'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-embedding/', }, + { - label: 'Output Field', - name: 'outputField', + label: 'Vector Field', + name: 'vectorField', type: 'string', - optional: false, - advanced: false, + helpText: + ' The name of the vector field in which to store the generated text embeddings.', + helpLink: + 'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-embedding/', }, ]; this.outputs = [ diff --git a/public/pages/workflow_detail/component_details/input_field_list.tsx b/public/pages/workflow_detail/component_details/input_field_list.tsx index 843ba8ea..92a11d29 100644 --- a/public/pages/workflow_detail/component_details/input_field_list.tsx +++ b/public/pages/workflow_detail/component_details/input_field_list.tsx @@ -34,7 +34,7 @@ export function InputFieldList(props: InputFieldListProps) { componentId={props.componentId} onFormChange={props.onFormChange} /> - + ); break; @@ -47,7 +47,7 @@ export function InputFieldList(props: InputFieldListProps) { componentId={props.componentId} onFormChange={props.onFormChange} /> - + ); break; @@ -59,6 +59,7 @@ export function InputFieldList(props: InputFieldListProps) { label={field.label} placeholder={field.placeholder || ''} /> + ); break; diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx index b043fb31..0af632da 100644 --- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx @@ -6,6 +6,7 @@ import React from 'react'; import { EuiFormRow, + EuiLink, EuiSuperSelect, EuiSuperSelectOption, EuiText, @@ -18,21 +19,6 @@ import { isFieldInvalid, } from '../../../../../common'; -// TODO: Should be fetched from global state. -// Need to have a way to determine where to fetch this dynamic data. -const existingIndices = [ - { - value: 'my-index-1', - inputDisplay: my-index-1, - disabled: false, - }, - { - value: 'my-index-2', - inputDisplay: my-index-2, - disabled: false, - }, -] as Array>; - interface SelectFieldProps { field: IComponentField; componentId: string; @@ -44,7 +30,15 @@ interface SelectFieldProps { * options. */ export function SelectField(props: SelectFieldProps) { - const options = existingIndices; + const selectOptions = (props.field.selectOptions || []).map( + (option) => + ({ + value: option, + inputDisplay: {option}, + disabled: false, + } as EuiSuperSelectOption) + ); + const formField = `${props.componentId}.${props.field.name}`; const { errors, touched } = useFormikContext(); @@ -52,9 +46,21 @@ export function SelectField(props: SelectFieldProps) { {({ field, form }: FieldProps) => { return ( - + + + Learn more + + + ) : undefined + } + helpText={props.field.helpText || undefined} + > { form.setFieldValue(formField, option); diff --git a/public/pages/workflow_detail/component_details/input_fields/text_field.tsx b/public/pages/workflow_detail/component_details/input_fields/text_field.tsx index d3866a7e..78ea27ec 100644 --- a/public/pages/workflow_detail/component_details/input_fields/text_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/text_field.tsx @@ -5,7 +5,7 @@ import React from 'react'; import { Field, FieldProps, useFormikContext } from 'formik'; -import { EuiFieldText, EuiFormRow } from '@elastic/eui'; +import { EuiFieldText, EuiFormRow, EuiLink, EuiText } from '@elastic/eui'; import { IComponentField, WorkspaceFormValues, @@ -34,6 +34,16 @@ export function TextField(props: TextFieldProps) { + + Learn more + + + ) : undefined + } + helpText={props.field.helpText || undefined} error={getFieldError(props.componentId, props.field.name, errors)} isInvalid={isFieldInvalid( props.componentId, diff --git a/public/utils/utils.ts b/public/utils/utils.ts index 8df05bc7..1a24f6ea 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -126,9 +126,13 @@ function getFieldSchema(field: IComponentField): Schema { break; } } - return field.optional - ? baseSchema.optional() - : baseSchema.required('Required'); + + // TODO: make optional schema if we support optional fields in the future + // return field.optional + // ? baseSchema.optional() + // : baseSchema.required('Required'); + + return baseSchema.required('Required'); } export function getStateOptions(): EuiFilterSelectItem[] { From 400a4dd5a41663825405e7201518cd0796de811f Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 2 Apr 2024 15:07:18 -0700 Subject: [PATCH 09/11] Onboard ML plugin and search models API; integrate into select field; Signed-off-by: Tyler Ohlsen --- common/constants.ts | 13 +++- common/interfaces.ts | 12 ++++ public/component_types/interfaces.ts | 3 +- .../transformer/text_embedding_transformer.ts | 2 +- .../input_fields/select_field.tsx | 38 ++++++++---- .../pages/workflow_detail/workflow_detail.tsx | 4 +- public/route_service.ts | 15 +++++ public/store/reducers/index.ts | 1 + public/store/reducers/models_reducer.ts | 62 +++++++++++++++++++ public/store/store.ts | 2 + server/cluster/index.ts | 1 + server/cluster/ml_plugin.ts | 27 ++++++++ server/plugin.ts | 8 ++- server/routes/helpers.ts | 22 ++++++- server/routes/index.ts | 1 + server/routes/ml_routes_service.ts | 61 ++++++++++++++++++ 16 files changed, 254 insertions(+), 18 deletions(-) create mode 100644 public/store/reducers/models_reducer.ts create mode 100644 server/cluster/ml_plugin.ts create mode 100644 server/routes/ml_routes_service.ts diff --git a/common/constants.ts b/common/constants.ts index e86b9d6c..f82a1167 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -6,13 +6,20 @@ export const PLUGIN_ID = 'flow-framework'; /** - * BACKEND/CLUSTER APIs + * BACKEND FLOW FRAMEWORK APIs */ export const FLOW_FRAMEWORK_API_ROUTE_PREFIX = '/_plugins/_flow_framework'; export const FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX = `${FLOW_FRAMEWORK_API_ROUTE_PREFIX}/workflow`; export const FLOW_FRAMEWORK_SEARCH_WORKFLOWS_ROUTE = `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}/_search`; export const FLOW_FRAMEWORK_SEARCH_WORKFLOW_STATE_ROUTE = `${FLOW_FRAMEWORK_WORKFLOW_ROUTE_PREFIX}/state/_search`; +/** + * BACKEND ML PLUGIN APIs + */ +export const ML_API_ROUTE_PREFIX = '/_plugins/_ml'; +export const ML_MODEL_ROUTE_PREFIX = `${ML_API_ROUTE_PREFIX}/models`; +export const ML_SEARCH_MODELS_ROUTE = `${ML_MODEL_ROUTE_PREFIX}/_search`; + /** * NODE APIs */ @@ -31,6 +38,10 @@ export const CREATE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/cre export const DELETE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/delete`; export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/presets`; +// ML Plugin node APIs +export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`; +export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`; + /** * MISCELLANEOUS */ diff --git a/common/interfaces.ts b/common/interfaces.ts index 2f3f0a74..eb412f1e 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -94,6 +94,14 @@ export enum USE_CASE { PROVISION = 'PROVISION', } +/** + ********** ML PLUGIN TYPES/INTERFACES ********** + */ +export type Model = { + id: string; + algorithm: string; +}; + /** ********** MISC TYPES/INTERFACES ************ */ @@ -116,3 +124,7 @@ export enum WORKFLOW_STATE { export type WorkflowDict = { [workflowId: string]: Workflow; }; + +export type ModelDict = { + [modelId: string]: Model; +}; diff --git a/public/component_types/interfaces.ts b/public/component_types/interfaces.ts index 51eda097..94e12952 100644 --- a/public/component_types/interfaces.ts +++ b/public/component_types/interfaces.ts @@ -11,6 +11,7 @@ import { COMPONENT_CATEGORY, COMPONENT_CLASS } from '../utils'; * ************ Types ************************* */ export type FieldType = 'string' | 'json' | 'select'; +export type SelectType = 'model'; // TODO: this may expand to more types in the future. Formik supports 'any' so we can too. // For now, limiting scope to expected types. export type FieldValue = string | {}; @@ -46,7 +47,7 @@ export interface IComponentField { placeholder?: string; helpText?: string; helpLink?: string; - selectOptions?: string[]; + selectType?: SelectType; } /** diff --git a/public/component_types/transformer/text_embedding_transformer.ts b/public/component_types/transformer/text_embedding_transformer.ts index f679ead3..28e07582 100644 --- a/public/component_types/transformer/text_embedding_transformer.ts +++ b/public/component_types/transformer/text_embedding_transformer.ts @@ -19,7 +19,7 @@ export class TextEmbeddingTransformer extends MLTransformer { label: 'Model ID', name: 'modelId', type: 'select', - selectOptions: ['model-1', 'test-model', 'model-2'], + selectType: 'model', helpText: 'The deployed text embedding model to use for embedding.', helpLink: 'https://opensearch.org/docs/latest/ml-commons-plugin/integrating-ml-models/#choosing-a-model', diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx index 0af632da..fc6448d8 100644 --- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx @@ -3,7 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Field, FieldProps, useFormikContext } from 'formik'; import { EuiFormRow, EuiLink, @@ -11,13 +13,13 @@ import { EuiSuperSelectOption, EuiText, } from '@elastic/eui'; -import { Field, FieldProps, useFormikContext } from 'formik'; import { IComponentField, WorkspaceFormValues, getInitialValue, isFieldInvalid, } from '../../../../../common'; +import { AppState } from '../../../../store'; interface SelectFieldProps { field: IComponentField; @@ -30,14 +32,21 @@ interface SelectFieldProps { * options. */ export function SelectField(props: SelectFieldProps) { - const selectOptions = (props.field.selectOptions || []).map( - (option) => - ({ - value: option, - inputDisplay: {option}, - disabled: false, - } as EuiSuperSelectOption) - ); + // Redux store state + // Initial store is fetched when loading base page. We don't + // re-fetch here as it could overload client-side if user clicks back and forth / + // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows + const models = useSelector((state: AppState) => state.models.models); + + // Options state + const [options, setOptions] = useState([]); + + // Populate options depending on the select type + useEffect(() => { + if (props.field.selectType === 'model' && models) { + setOptions(Object.keys(models)); + } + }, [models]); const formField = `${props.componentId}.${props.field.name}`; const { errors, touched } = useFormikContext(); @@ -60,7 +69,14 @@ export function SelectField(props: SelectFieldProps) { helpText={props.field.helpText || undefined} > + ({ + value: option, + inputDisplay: {option}, + disabled: false, + } as EuiSuperSelectOption) + )} valueOfSelected={field.value || getInitialValue(props.field.type)} onChange={(option) => { form.setFieldValue(formField, option); diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 527ce0eb..43664bf5 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -12,7 +12,7 @@ import { EuiPage, EuiPageBody } from '@elastic/eui'; import { BREADCRUMBS } from '../../utils'; import { getCore } from '../../services'; import { WorkflowDetailHeader } from './components'; -import { AppState, searchWorkflows } from '../../store'; +import { AppState, searchModels, searchWorkflows } from '../../store'; import { ResizableWorkspace } from './workspace'; import { Launches } from './launches'; import { Prototype } from './prototype'; @@ -97,11 +97,13 @@ export function WorkflowDetail(props: WorkflowDetailProps) { // On initial load: // - fetch workflow, if there is an existing workflow ID + // - fetch available models as their IDs may be used when building flows useEffect(() => { if (!isNewWorkflow) { // TODO: can optimize to only fetch a single workflow dispatch(searchWorkflows({ query: { match_all: {} } })); } + dispatch(searchModels({ query: { match_all: {} } })); }, []); const tabs = [ diff --git a/public/route_service.ts b/public/route_service.ts index 334d0ddb..78faddd9 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -12,6 +12,7 @@ import { GET_WORKFLOW_STATE_NODE_API_PATH, SEARCH_WORKFLOWS_NODE_API_PATH, GET_PRESET_WORKFLOWS_NODE_API_PATH, + SEARCH_MODELS_NODE_API_PATH, } from '../common'; /** @@ -32,6 +33,7 @@ export interface RouteService { deleteWorkflow: (workflowId: string) => Promise; getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; + searchModels: (body: {}) => Promise; } export function configureRoutes(core: CoreStart): RouteService { @@ -112,5 +114,18 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + searchModels: async (body: {}) => { + try { + const response = await core.http.post<{ respString: string }>( + SEARCH_MODELS_NODE_API_PATH, + { + body: JSON.stringify(body), + } + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, }; } diff --git a/public/store/reducers/index.ts b/public/store/reducers/index.ts index 522987d9..ef82cf2c 100644 --- a/public/store/reducers/index.ts +++ b/public/store/reducers/index.ts @@ -7,3 +7,4 @@ export * from './workspace_reducer'; export * from './opensearch_reducer'; export * from './workflows_reducer'; export * from './presets_reducer'; +export * from './models_reducer'; diff --git a/public/store/reducers/models_reducer.ts b/public/store/reducers/models_reducer.ts new file mode 100644 index 00000000..2940844e --- /dev/null +++ b/public/store/reducers/models_reducer.ts @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; +import { ModelDict } from '../../../common'; +import { HttpFetchError } from '../../../../../src/core/public'; +import { getRouteService } from '../../services'; + +const initialState = { + loading: false, + errorMessage: '', + models: {} as ModelDict, +}; + +const MODELS_ACTION_PREFIX = 'models'; +const SEARCH_MODELS_ACTION = `${MODELS_ACTION_PREFIX}/searchModels`; + +export const searchModels = createAsyncThunk( + SEARCH_MODELS_ACTION, + async (body: {}, { rejectWithValue }) => { + const response: any | HttpFetchError = await getRouteService().searchModels( + body + ); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error searching models: ' + response.body.message + ); + } else { + return response; + } + } +); + +const modelsSlice = createSlice({ + name: 'models', + initialState, + reducers: {}, + extraReducers: (builder) => { + builder + // Pending states: set state consistently across all actions + .addCase(searchModels.pending, (state, action) => { + state.loading = true; + state.errorMessage = ''; + }) + + .addCase(searchModels.fulfilled, (state, action) => { + const { models } = action.payload as { models: ModelDict }; + state.models = models; + state.loading = false; + state.errorMessage = ''; + }) + + .addCase(searchModels.rejected, (state, action) => { + state.errorMessage = action.payload as string; + state.loading = false; + }); + }, +}); + +export const modelsReducer = modelsSlice.reducer; diff --git a/public/store/store.ts b/public/store/store.ts index fa906b99..06d7d2ea 100644 --- a/public/store/store.ts +++ b/public/store/store.ts @@ -10,12 +10,14 @@ import { opensearchReducer, workflowsReducer, presetsReducer, + modelsReducer, } from './reducers'; const rootReducer = combineReducers({ workspace: workspaceReducer, workflows: workflowsReducer, presets: presetsReducer, + models: modelsReducer, opensearch: opensearchReducer, }); diff --git a/server/cluster/index.ts b/server/cluster/index.ts index 5ff70614..a29304ea 100644 --- a/server/cluster/index.ts +++ b/server/cluster/index.ts @@ -4,3 +4,4 @@ */ export * from './flow_framework_plugin'; +export * from './ml_plugin'; diff --git a/server/cluster/ml_plugin.ts b/server/cluster/ml_plugin.ts new file mode 100644 index 00000000..4601a9dd --- /dev/null +++ b/server/cluster/ml_plugin.ts @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ML_SEARCH_MODELS_ROUTE } from '../../common'; + +/** + * Used during the plugin's setup() lifecycle phase to register various client actions + * representing ML plugin APIs. These are then exposed and used on the + * server-side when processing node APIs - see server/routes/ml_routes_service + * for examples. + */ +export function mlPlugin(Client: any, config: any, components: any) { + const ca = components.clientAction.factory; + + Client.prototype.mlClient = components.clientAction.namespaceFactory(); + const mlClient = Client.prototype.mlClient.prototype; + + mlClient.searchModels = ca({ + url: { + fmt: ML_SEARCH_MODELS_ROUTE, + }, + needBody: true, + method: 'POST', + }); +} diff --git a/server/plugin.ts b/server/plugin.ts index 2772d5aa..62c8ae27 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -11,7 +11,7 @@ import { Logger, } from '../../../src/core/server'; import { first } from 'rxjs/operators'; -import { flowFrameworkPlugin } from './cluster'; +import { flowFrameworkPlugin, mlPlugin } from './cluster'; import { FlowFrameworkDashboardsPluginSetup, FlowFrameworkDashboardsPluginStart, @@ -21,6 +21,8 @@ import { registerFlowFrameworkRoutes, OpenSearchRoutesService, FlowFrameworkRoutesService, + registerMLRoutes, + MLRoutesService, } from './routes'; import { ILegacyClusterClient } from '../../../src/core/server/'; @@ -50,17 +52,19 @@ export class FlowFrameworkDashboardsPlugin const client: ILegacyClusterClient = core.opensearch.legacy.createClient( 'flow_framework', { - plugins: [flowFrameworkPlugin], + plugins: [flowFrameworkPlugin, mlPlugin], ...globalConfig.opensearch, } ); const opensearchRoutesService = new OpenSearchRoutesService(client); const flowFrameworkRoutesService = new FlowFrameworkRoutesService(client); + const mlRoutesService = new MLRoutesService(client); // Register server side APIs with the corresponding service functions registerOpenSearchRoutes(router, opensearchRoutesService); registerFlowFrameworkRoutes(router, flowFrameworkRoutesService); + registerMLRoutes(router, mlRoutesService); return {}; } diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index 8fddfe8f..5b912b49 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -3,7 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { WORKFLOW_STATE, Workflow, WorkflowDict } from '../../common'; +import { + Model, + ModelDict, + WORKFLOW_STATE, + Workflow, + WorkflowDict, +} from '../../common'; // OSD does not provide an interface for this response, but this is following the suggested // implementations. To prevent typescript complaining, leaving as loosely-typed 'any' @@ -61,3 +67,17 @@ export function getWorkflowsFromResponses( }); return workflowDict; } + +export function getModelsFromResponses(modelHits: any[]): ModelDict { + const modelDict = {} as ModelDict; + modelHits.forEach((modelHit: any) => { + const modelId = modelHit._source?.model_id; + // in case of schema changes from ML plugin, this may crash. That is ok, as the error + // produced will help expose the root cause + modelDict[modelId] = { + id: modelId, + algorithm: modelHit._source?.algorithm, + } as Model; + }); + return modelDict; +} diff --git a/server/routes/index.ts b/server/routes/index.ts index 8b55b8cb..db0e21c8 100644 --- a/server/routes/index.ts +++ b/server/routes/index.ts @@ -5,3 +5,4 @@ export * from './opensearch_routes_service'; export * from './flow_framework_routes_service'; +export * from './ml_routes_service'; diff --git a/server/routes/ml_routes_service.ts b/server/routes/ml_routes_service.ts new file mode 100644 index 00000000..874d84d3 --- /dev/null +++ b/server/routes/ml_routes_service.ts @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { schema } from '@osd/config-schema'; +import { + IRouter, + IOpenSearchDashboardsResponse, + RequestHandlerContext, + OpenSearchDashboardsRequest, + OpenSearchDashboardsResponseFactory, +} from '../../../../src/core/server'; +import { SEARCH_MODELS_NODE_API_PATH } from '../../common'; +import { generateCustomError, getModelsFromResponses } from './helpers'; + +/** + * Server-side routes to process ml-plugin-related node API calls and execute the + * corresponding API calls against the OpenSearch cluster. + */ +export function registerMLRoutes( + router: IRouter, + mlRoutesService: MLRoutesService +): void { + router.post( + { + path: SEARCH_MODELS_NODE_API_PATH, + validate: { + body: schema.any(), + }, + }, + mlRoutesService.searchModels + ); +} + +export class MLRoutesService { + private client: any; + + constructor(client: any) { + this.client = client; + } + + searchModels = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const body = req.body; + try { + const modelsResponse = await this.client + .asScoped(req) + .callAsCurrentUser('mlClient.searchModels', { body }); + const modelHits = modelsResponse.hits.hits as any[]; + const modelDict = getModelsFromResponses(modelHits); + + return res.ok({ body: { models: modelDict } }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; +} From 854b4da517e74d8b8ae1df33e79936e01ced834c Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 2 Apr 2024 15:39:47 -0700 Subject: [PATCH 10/11] add reducer comments Signed-off-by: Tyler Ohlsen --- public/store/reducers/models_reducer.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/store/reducers/models_reducer.ts b/public/store/reducers/models_reducer.ts index 2940844e..a43ccb79 100644 --- a/public/store/reducers/models_reducer.ts +++ b/public/store/reducers/models_reducer.ts @@ -39,19 +39,19 @@ const modelsSlice = createSlice({ reducers: {}, extraReducers: (builder) => { builder - // Pending states: set state consistently across all actions + // Pending states .addCase(searchModels.pending, (state, action) => { state.loading = true; state.errorMessage = ''; }) - + // Fulfilled states .addCase(searchModels.fulfilled, (state, action) => { const { models } = action.payload as { models: ModelDict }; state.models = models; state.loading = false; state.errorMessage = ''; }) - + // Rejected states .addCase(searchModels.rejected, (state, action) => { state.errorMessage = action.payload as string; state.loading = false; From 4c109fe1040e233b65820aeac74c87d4267435ea Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 3 Apr 2024 10:34:05 -0700 Subject: [PATCH 11/11] address minor comments Signed-off-by: Tyler Ohlsen --- .../component_details/component_inputs.tsx | 24 ++++++++----------- .../component_details/input_field_list.tsx | 8 ++++--- .../input_fields/select_field.tsx | 2 +- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/public/pages/workflow_detail/component_details/component_inputs.tsx b/public/pages/workflow_detail/component_details/component_inputs.tsx index 5f49895b..6e8ad9b2 100644 --- a/public/pages/workflow_detail/component_details/component_inputs.tsx +++ b/public/pages/workflow_detail/component_details/component_inputs.tsx @@ -60,20 +60,16 @@ export function ComponentInputs(props: ComponentInputsProps) { setSelectedTabId={setSelectedTabId} /> - {selectedTabId === TAB.NEW && ( - - )} - {selectedTabId === TAB.EXISTING && ( - - )} + + ); } diff --git a/public/pages/workflow_detail/component_details/input_field_list.tsx b/public/pages/workflow_detail/component_details/input_field_list.tsx index 92a11d29..ca13d59c 100644 --- a/public/pages/workflow_detail/component_details/input_field_list.tsx +++ b/public/pages/workflow_detail/component_details/input_field_list.tsx @@ -19,6 +19,8 @@ interface InputFieldListProps { onFormChange: () => void; } +const INPUT_FIELD_SPACER_SIZE = 'm'; + export function InputFieldList(props: InputFieldListProps) { const inputFields = props.componentFields || []; return ( @@ -34,7 +36,7 @@ export function InputFieldList(props: InputFieldListProps) { componentId={props.componentId} onFormChange={props.onFormChange} /> - + ); break; @@ -47,7 +49,7 @@ export function InputFieldList(props: InputFieldListProps) { componentId={props.componentId} onFormChange={props.onFormChange} /> - + ); break; @@ -59,7 +61,7 @@ export function InputFieldList(props: InputFieldListProps) { label={field.label} placeholder={field.placeholder || ''} /> - + ); break; diff --git a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx index fc6448d8..008ece7f 100644 --- a/public/pages/workflow_detail/component_details/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/component_details/input_fields/select_field.tsx @@ -4,7 +4,7 @@ */ import React, { useEffect, useState } from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { useSelector } from 'react-redux'; import { Field, FieldProps, useFormikContext } from 'formik'; import { EuiFormRow,