Add server-side workflow template library (#112) (#113)

Signed-off-by: Tyler Ohlsen <[email protected]>
(cherry picked from commit 83835c2)

Co-authored-by: Tyler Ohlsen <[email protected]>

common/constants.ts

@@ -29,6 +29,7 @@ export const SEARCH_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/se
 export const GET_WORKFLOW_STATE_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/state`;
 export const CREATE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/create`;
 export const DELETE_WORKFLOW_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/delete`;
+export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH}/presets`;
 
 /**
  * MISCELLANEOUS

common/index.ts

@@ -5,5 +5,6 @@
 
 export * from './constants';
 export * from './interfaces';
+export * from './utils';
 export * from '../public/component_types';
 export * from '../public/utils';

common/interfaces.ts

@@ -40,7 +40,9 @@ export type WorkspaceFlowState = {
 
 export type TemplateNode = {
   id: string;
-  inputs: {};
+  type: string;
+  previous_node_inputs?: Map<string, any>;
+  user_inputs?: Map<string, any>;
 };
 
 export type TemplateEdge = {
@@ -49,32 +51,30 @@ export type TemplateEdge = {
 };
 
 export type TemplateFlow = {
-  userParams: {};
+  user_params?: Map<string, any>;
   nodes: TemplateNode[];
-  edges: TemplateEdge[];
+  edges?: TemplateEdge[];
 };
 
 export type TemplateFlows = {
   provision: TemplateFlow;
-  ingest: TemplateFlow;
-  query: TemplateFlow;
 };
 
-export type UseCaseTemplate = {
-  type: string;
+// A stateless template of a workflow
+export type WorkflowTemplate = {
   name: string;
   description: string;
-  userInputs: {};
+  use_case: USE_CASE;
+  // TODO: finalize on version type when that is implemented
+  // https://github.com/opensearch-project/flow-framework/issues/526
+  version: any;
   workflows: TemplateFlows;
 };
 
-export type Workflow = {
+// An instance of a workflow based on a workflow template
+export type Workflow = WorkflowTemplate & {
   // won't exist until created in backend
   id?: string;
-  name: string;
-  useCase: string;
-  template: UseCaseTemplate;
-  description?: string;
   // ReactFlow state may not exist if a workflow is created via API/backend-only.
   workspaceFlowState?: WorkspaceFlowState;
   // won't exist until created in backend
@@ -86,8 +86,7 @@ export type Workflow = {
 };
 
 export enum USE_CASE {
-  SEMANTIC_SEARCH = 'semantic_search',
-  CUSTOM = 'custom',
+  PROVISION = 'PROVISION',
 }
 
 /**

common/utils.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  WorkspaceFlowState,
+  ReactFlowComponent,
+  initComponentData,
+  TextEmbeddingTransformer,
+  KnnIndexer,
+  generateId,
+  ReactFlowEdge,
+  TemplateFlows,
+  WorkflowTemplate,
+} from './';
+
+// TODO: implement this and remove hardcoded return values
+/**
+ * Converts a ReactFlow workspace flow to a backend-compatible set of ingest and/or search sub-workflows,
+ * along with a provision sub-workflow if resources are to be created.
+ */
+export function toTemplateFlows(
+  workspaceFlow: WorkspaceFlowState
+): TemplateFlows {
+  return {
+    provision: {
+      user_params: {} as Map<string, any>,
+      nodes: [],
+      edges: [],
+    },
+  };
+}
+
+// TODO: implement this and remove hardcoded return values
+/**
+ * Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of
+ * ReactFlow nodes and edges
+ */
+export function toWorkspaceFlow(
+  templateFlows: TemplateFlows
+): WorkspaceFlowState {
+  const id1 = generateId('text_embedding_processor');
+  const id2 = generateId('text_embedding_processor');
+  const id3 = generateId('knn_index');
+  const dummyNodes = [
+    {
+      id: id1,
+      position: { x: 0, y: 500 },
+      data: initComponentData(new TextEmbeddingTransformer().toObj(), id1),
+      type: 'customComponent',
+    },
+    {
+      id: id2,
+      position: { x: 0, y: 200 },
+      data: initComponentData(new TextEmbeddingTransformer().toObj(), id2),
+      type: 'customComponent',
+    },
+    {
+      id: id3,
+      position: { x: 500, y: 500 },
+      data: initComponentData(new KnnIndexer().toObj(), id3),
+      type: 'customComponent',
+    },
+  ] as ReactFlowComponent[];
+
+  return {
+    nodes: dummyNodes,
+    edges: [] as ReactFlowEdge[],
+  };
+}
+
+// TODO: implement this
+/**
+ * Validates the UI workflow state.
+ * Note we don't have to validate connections since that is done via input/output handlers.
+ */
+export function validateWorkspaceFlow(
+  workspaceFlow: WorkspaceFlowState
+): boolean {
+  return true;
+}
+
+// TODO: implement this
+/**
+ * Validates the backend template. May be used when parsing persisted templates on server-side,
+ * or when importing/exporting on the UI.
+ */
+export function validateWorkflowTemplate(
+  workflowTemplate: WorkflowTemplate
+): boolean {
+  return true;
+}

public/pages/workflow_detail/utils/utils.ts

@@ -5,10 +5,10 @@
 
 import {
   WorkspaceFlowState,
-  UseCaseTemplate,
   Workflow,
-  USE_CASE,
   ReactFlowComponent,
+  toTemplateFlows,
+  validateWorkspaceFlow,
 } from '../../../../common';
 
 export function saveWorkflow(workflow: Workflow, rfInstance: any): void {
@@ -19,12 +19,12 @@ export function saveWorkflow(workflow: Workflow, rfInstance: any): void {
     nodes: processNodes(curFlowState.nodes),
   };
 
-  const isValid = validateFlowState(curFlowState);
+  const isValid = validateWorkspaceFlow(curFlowState);
   if (isValid) {
     const updatedWorkflow = {
       ...workflow,
       workspaceFlowState: curFlowState,
-      template: generateUseCaseTemplate(curFlowState),
+      workflows: toTemplateFlows(curFlowState),
     } as Workflow;
     if (workflow.id) {
       // TODO: implement connection to update workflow API
@@ -36,42 +36,6 @@ export function saveWorkflow(workflow: Workflow, rfInstance: any): void {
   }
 }
 
-// TODO: implement this. Need more info on UX side to finalize what we need
-// to persist, what validation to do, etc.
-// Note we don't have to validate connections since that is done via input/output handlers.
-function validateFlowState(flowState: WorkspaceFlowState): boolean {
-  return true;
-}
-
-// TODO: implement this
-function generateUseCaseTemplate(
-  flowState: WorkspaceFlowState
-): UseCaseTemplate {
-  return {
-    name: 'example-name',
-    description: 'example description',
-    type: USE_CASE.SEMANTIC_SEARCH,
-    userInputs: {},
-    workflows: {
-      provision: {
-        userParams: {},
-        nodes: [],
-        edges: [],
-      },
-      ingest: {
-        userParams: {},
-        nodes: [],
-        edges: [],
-      },
-      query: {
-        userParams: {},
-        nodes: [],
-        edges: [],
-      },
-    },
-  } as UseCaseTemplate;
-}
-
 // Process the raw ReactFlow nodes to only persist the fields we need
 function processNodes(nodes: ReactFlowComponent[]): ReactFlowComponent[] {
   return nodes

public/pages/workflow_detail/workflow_detail.tsx

@@ -105,8 +105,15 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
       // TODO: can optimize to only fetch a single workflow
       dispatch(searchWorkflows({ query: { match_all: {} } }));
     }
-    window.onbeforeunload = (e) =>
-      isDirty || isNewWorkflow ? true : undefined;
+
+    // TODO: below has the following issue:
+    // 1. user starts to create new unsaved workflow changes
+    // 2. user navigates to other parts of the plugin without refreshing - no warning happens
+    // 3. user refreshes at any later time: if isDirty is still true, shows browser warning
+    // tune to only handle the check if still on the workflow details page, or consider adding a check / warning
+    // if navigating away from the details page without refreshing (where it is currently not being triggered)
+    // window.onbeforeunload = (e) =>
+    //   isDirty || isNewWorkflow ? true : undefined;
   }, []);
 
   const tabs = [

public/pages/workflow_detail/workspace/workspace.tsx

@@ -21,9 +21,9 @@ import {
   IComponentData,
   ReactFlowComponent,
   Workflow,
+  toWorkspaceFlow,
 } from '../../../../common';
 import { generateId, initComponentData } from '../../../utils';
-import { getCore } from '../../../services';
 import { WorkspaceComponent } from '../workspace_component';
 import { DeletableEdge } from '../workspace_edge';
 
@@ -119,16 +119,19 @@ export function Workspace(props: WorkspaceProps) {
   // Initialization. Set the nodes and edges to an existing workflow,
   // if applicable.
   useEffect(() => {
-    const workflow = props.workflow;
+    const workflow = { ...props.workflow };
     if (workflow) {
-      if (workflow.workspaceFlowState) {
-        setNodes(workflow.workspaceFlowState.nodes);
-        setEdges(workflow.workspaceFlowState.edges);
-      } else {
-        getCore().notifications.toasts.addWarning(
-          `There is no configured UI flow for workflow: ${workflow.name}`
+      if (!workflow.workspaceFlowState) {
+        // No existing workspace state. This could be due to it being a backend-only-created
+        // workflow, or a new, unsaved workflow
+        // @ts-ignore
+        workflow.workspaceFlowState = toWorkspaceFlow(workflow.workflows);
+        console.debug(
+          `There is no saved UI flow for workflow: ${workflow.name}. Generating a default one.`
         );
       }
+      setNodes(workflow.workspaceFlowState.nodes);
+      setEdges(workflow.workspaceFlowState.edges);
     }
   }, [props.workflow]);