Parse and persist UI metadata on editor page (#125)

Signed-off-by: Tyler Ohlsen <[email protected]>

.github/workflows/build-and-test.yml

@@ -45,8 +45,6 @@ jobs:
           su `id -un 1000` -c "source $NVM_DIR/nvm.sh && nvm use && node -v && yarn -v &&
                                cd ./plugins/dashboards-flow-framework &&
                                whoami && yarn osd bootstrap && yarn build && yarn run test:jest --coverage"
-      - name: Uploads coverage
-        uses: codecov/codecov-action@v1
 
   # TODO: once github actions supports windows and macos docker containers, we can
   # merge these in to the above step's matrix, including adding windows support
@@ -98,6 +96,4 @@ jobs:
         run: |
           cd OpenSearch-Dashboards/plugins/dashboards-flow-framework
           yarn run test:jest --coverage
-      - name: Uploads coverage
-        uses: codecov/codecov-action@v1
 

codecov.yml

@@ -0,0 +1,17 @@
+# disable tracking status entirely until UT is added.
+# tracking issue: https://github.com/opensearch-project/dashboards-flow-framework/issues/95 
+coverage:
+  # displays different colors depending on below, between, or above the range
+  range: 50..90
+  status:
+    project:
+      enabled: no
+      default: 
+        target: auto
+        # allows 5% coverage reduction without failing
+        threshold: 5%
+    patch: no
+    changes: no
+
+# disable comments in PRs
+comment: no

package.json

@@ -13,17 +13,13 @@
     "osd": "../../scripts/use_node ../../scripts/osd",
     "opensearch": "../../scripts/use_node ../../scripts/opensearch",
     "lint:es": "../../scripts/use_node ../../scripts/eslint -c eslintrc.json",
-    "lint:es:precommit": "yarn lint:es common/* public/* server/*",
     "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
     "build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/opensearch-project/dashboards-flow-framework.git"
   },
-  "pre-commit": [
-    "lint:es:precommit"
-  ],
   "lint-staged": {
     "*.{ts,tsx,js,jsx,json,css,md}": [
       "prettier --write",
@@ -35,8 +31,6 @@
     "reactflow": "^11.8.3",
     "yup": "^1.3.2"
   },
-  "devDependencies": {
-    "pre-commit": "^1.2.2"
-  },
+  "devDependencies": {},
   "resolutions": {}
 }

public/component_types/indexer/indexer.ts

@@ -31,19 +31,19 @@ export class Indexer extends BaseComponent {
     this.fields = [
       {
         label: 'Index Name',
-        name: 'indexName',
+        id: 'indexName',
         type: 'select',
       },
     ];
     this.createFields = [
       {
         label: 'Index Name',
-        name: 'indexName',
+        id: 'indexName',
         type: 'string',
       },
       // {
       //   label: 'Mappings',
-      //   name: 'indexMappings',
+      //   id: 'indexMappings',
       //   type: 'json',
       //   placeholder: 'Enter an index mappings JSON blob...',
       // },

public/component_types/indexer/knn_indexer.ts

@@ -19,7 +19,7 @@ export class KnnIndexer extends Indexer {
       // TODO: finalize what to expose / what to have for defaults here
       // {
       //   label: 'K-NN Settings',
-      //   name: 'knnSettings',
+      //   id: 'knnSettings',
       //   type: 'json',
       //   placeholder: 'Enter K-NN settings JSON blob...',
       // },

public/component_types/interfaces.ts

@@ -42,7 +42,7 @@ export interface IComponentInput {
 export interface IComponentField {
   label: string;
   type: FieldType;
-  name: string;
+  id: string;
   value?: FieldValue;
   placeholder?: string;
   helpText?: string;

public/component_types/transformer/text_embedding_transformer.ts

@@ -17,7 +17,7 @@ export class TextEmbeddingTransformer extends MLTransformer {
     this.createFields = [
       {
         label: 'Model ID',
-        name: 'modelId',
+        id: 'modelId',
         type: 'select',
         selectType: 'model',
         helpText: 'The deployed text embedding model to use for embedding.',
@@ -26,7 +26,7 @@ export class TextEmbeddingTransformer extends MLTransformer {
       },
       {
         label: 'Input Field',
-        name: 'inputField',
+        id: 'inputField',
         type: 'string',
         helpText:
           'The name of the field from which to obtain text for generating text embeddings.',
@@ -36,7 +36,7 @@ export class TextEmbeddingTransformer extends MLTransformer {
 
       {
         label: 'Vector Field',
-        name: 'vectorField',
+        id: 'vectorField',
         type: 'string',
         helpText:
           '	The name of the vector field in which to store the generated text embeddings.',

public/pages/workflow_detail/component_details/input_fields/select_field.tsx

@@ -48,7 +48,7 @@ export function SelectField(props: SelectFieldProps) {
     }
   }, [models]);
 
-  const formField = `${props.componentId}.${props.field.name}`;
+  const formField = `${props.componentId}.${props.field.id}`;
   const { errors, touched } = useFormikContext<WorkspaceFormValues>();
 
   return (
@@ -84,7 +84,7 @@ export function SelectField(props: SelectFieldProps) {
               }}
               isInvalid={isFieldInvalid(
                 props.componentId,
-                props.field.name,
+                props.field.id,
                 errors,
                 touched
               )}

public/pages/workflow_detail/component_details/input_fields/text_field.tsx

@@ -24,7 +24,7 @@ interface TextFieldProps {
  * An input field for a component where users input plaintext
  */
 export function TextField(props: TextFieldProps) {
-  const formField = `${props.componentId}.${props.field.name}`;
+  const formField = `${props.componentId}.${props.field.id}`;
   const { errors, touched } = useFormikContext<WorkspaceFormValues>();
 
   return (
@@ -44,10 +44,10 @@ export function TextField(props: TextFieldProps) {
               ) : undefined
             }
             helpText={props.field.helpText || undefined}
-            error={getFieldError(props.componentId, props.field.name, errors)}
+            error={getFieldError(props.componentId, props.field.id, errors)}
             isInvalid={isFieldInvalid(
               props.componentId,
-              props.field.name,
+              props.field.id,
               errors,
               touched
             )}

public/pages/workflow_detail/workspace/resizable_workspace.tsx

@@ -34,6 +34,7 @@ import {
   DEFAULT_NEW_WORKFLOW_DESCRIPTION,
   USE_CASE,
   WORKFLOW_STATE,
+  processNodes,
 } from '../../../../common';
 import {
   AppState,
@@ -47,7 +48,6 @@ import {
 } from '../../../store';
 import { Workspace } from './workspace';
 import { ComponentDetails } from '../component_details';
-import { processNodes } from '../utils';
 
 // styling
 import './workspace-styles.scss';
@@ -147,30 +147,36 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
   // Metadata fields (name/description/use_case/etc.) may not exist if the user
   // cold reloads the page on a new, unsaved workflow.
   useEffect(() => {
-    let workflowCopy = { ...props.workflow } as Workflow;
-    if (!workflowCopy.ui_metadata || !workflowCopy.ui_metadata.workspaceFlow) {
-      workflowCopy.ui_metadata = {
-        ...(workflowCopy.ui_metadata || {}),
-        workspaceFlow: toWorkspaceFlow(workflowCopy.workflows),
-      };
-      console.debug(
-        `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.`
-      );
-    }
+    if (props.workflow) {
+      let workflowCopy = { ...props.workflow } as Workflow;
+      if (
+        !workflowCopy.ui_metadata ||
+        !workflowCopy.ui_metadata.workspaceFlow
+      ) {
+        workflowCopy.ui_metadata = {
+          ...(workflowCopy.ui_metadata || {}),
+          workspaceFlow: toWorkspaceFlow(workflowCopy.workflows),
+        };
+        console.debug(
+          `There is no saved UI flow for workflow: ${workflowCopy.name}. Generating a default one.`
+        );
+      }
 
-    // TODO: tune some of the defaults, like use_case and version as these will change
-    workflowCopy = {
-      ...workflowCopy,
-      name: workflowCopy.name || DEFAULT_NEW_WORKFLOW_NAME,
-      description: workflowCopy.description || DEFAULT_NEW_WORKFLOW_DESCRIPTION,
-      use_case: workflowCopy.use_case || USE_CASE.PROVISION,
-      version: workflowCopy.version || {
-        template: '1.0.0',
-        compatibility: ['2.12.0', '3.0.0'],
-      },
-    };
+      // TODO: tune some of the defaults, like use_case and version as these will change
+      workflowCopy = {
+        ...workflowCopy,
+        name: workflowCopy.name || DEFAULT_NEW_WORKFLOW_NAME,
+        description:
+          workflowCopy.description || DEFAULT_NEW_WORKFLOW_DESCRIPTION,
+        use_case: workflowCopy.use_case || USE_CASE.PROVISION,
+        version: workflowCopy.version || {
+          template: '1.0.0',
+          compatibility: ['2.12.0', '3.0.0'],
+        },
+      };
 
-    setWorkflow(workflowCopy);
+      setWorkflow(workflowCopy);
+    }
   }, [props.workflow]);
 
   // Hook to updated the selected ReactFlow component
@@ -260,7 +266,7 @@ export function ResizableWorkspace(props: ResizableWorkspaceProps) {
         let curFlowState = reactFlowInstance.toObject() as WorkspaceFlowState;
         curFlowState = {
           ...curFlowState,
-          nodes: processNodes(curFlowState.nodes),
+          nodes: processNodes(curFlowState.nodes, formikProps.values),
         };
         if (validateWorkspaceFlow(curFlowState)) {
           setFlowValidOnSubmit(true);

public/utils/utils.ts

@@ -15,6 +15,7 @@ import {
   IComponentField,
   WorkspaceFormValues,
   WORKFLOW_STATE,
+  ReactFlowComponent,
 } from '../../common';
 
 // Append 16 random characters
@@ -48,18 +49,27 @@ export function initComponentData(
 export function componentDataToFormik(data: IComponentData): FormikValues {
   const formikValues = {} as FormikValues;
   data.createFields?.forEach((field) => {
-    formikValues[field.name] = field.value || getInitialValue(field.type);
+    formikValues[field.id] = field.value || getInitialValue(field.type);
   });
   return formikValues;
 }
 
+// TODO: below, we are hardcoding to only persisting and validating create fields.
+// If we support both, we will need to dynamically update.
+// Injecting the current form values into the component data
 export function formikToComponentData(
-  data: IComponentData,
-  values: FormikValues
+  origData: IComponentData,
+  formValues: FormikValues
 ): IComponentData {
-  // TODO: populate data.fields with updated values based on the formik values
-  // We will need this when submitting to the backend.
-  return data;
+  return {
+    ...origData,
+    createFields: origData.createFields?.map(
+      (createField: IComponentField) => ({
+        ...createField,
+        value: formValues[createField.id],
+      })
+    ),
+  } as IComponentData;
 }
 
 // Helper fn to get an initial value based on the field type
@@ -97,6 +107,24 @@ export function getFieldError(
   return errors[componentId]?.[fieldName] as string | undefined;
 }
 
+// Process the raw ReactFlow nodes.
+// De-select them all, and propagate the form data to the internal node data
+export function processNodes(
+  nodes: ReactFlowComponent[],
+  formValues: WorkspaceFormValues
+): ReactFlowComponent[] {
+  return nodes.map((node: ReactFlowComponent) => {
+    return {
+      ...node,
+      selected: false,
+      data: formikToComponentData(
+        { ...node.data, selected: false },
+        formValues[node.id]
+      ),
+    };
+  });
+}
+
 /*
  **************** Yup (validation) utils **********************
  */
@@ -106,7 +134,7 @@ export function getFieldError(
 export function getComponentSchema(data: IComponentData): ObjectSchema<any> {
   const schemaObj = {} as { [key: string]: Schema };
   data.createFields?.forEach((field) => {
-    schemaObj[field.name] = getFieldSchema(field);
+    schemaObj[field.id] = getFieldSchema(field);
   });
   return yup.object(schemaObj);
 }

server/routes/helpers.ts

@@ -37,6 +37,7 @@ function toWorkflowObj(workflowHit: any): Workflow {
     description: hitSource.description || '',
     version: hitSource.version,
     workflows: hitSource.workflows,
+    ui_metadata: hitSource.ui_metadata,
     lastUpdated: hitSource.last_updated_time,
     lastLaunched: hitSource.last_provisioned_time,
   } as Workflow;