add globals, fix openai temp, remove parameters

pufflyai · Sep 7, 2023 · 14ab633 · 14ab633
1 parent 8da830a
commit 14ab633
Show file tree

Hide file tree

Showing 43 changed files with 258 additions and 186 deletions.
diff --git a/.gitignore b/.gitignore
@@ -132,3 +132,6 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+# docusaurus
+build
diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/constants.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/constants.ts
@@ -1,12 +1,27 @@
-import { NextNode, Node, ParamValueMap } from "@pufflig/ps-types";
+import { Execute, GetTargets, NextNode, Node, ParamValueMap } from "@pufflig/ps-types";
 
 export const delimiterStart = "${{ps:ref:" as const;
 export const delimiterEnd = "}}" as const;
 export const executionPrefix = "exec:";
 
-export const identity = (i: ParamValueMap, _: Partial<ParamValueMap>) => i;
+/**
+ * Placeholder function for executing a node.
+ * Simply returns the nodes inputs.
+ * @param input
+ * @param _
+ * @returns
+ */
+export const identity: Execute<ParamValueMap> = async (input, _) => input;
+
+/**
+ * Given a node, returns the default targets for the node.
+ * The default target is the first executable output.
+ * @param node
+ * @returns
+ */
 export const getDefaultTargets =
-  (node: Node) => (_input: ParamValueMap, _prev: ParamValueMap, results: ParamValueMap) => {
+  (node: Node): GetTargets<ParamValueMap> =>
+  async (_, results) => {
     if (!node.execution?.outputs?.[0]?.id) return [];
     return [
       {

diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/runFlow.test.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/runFlow.test.ts
@@ -458,7 +458,6 @@ test("a node can run its children multiple times 2", async () => {
     "n1",
     {},
     {
-      logLevel: "debug",
       onNodeRunComplete,
       onNodeInputUpdate,
       onNodeRunError,

diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/runFlow.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/runFlow.ts
@@ -91,7 +91,8 @@ export async function runFlow(flow: Flow, nodeId: string, input: Record<string,
     let result: Record<string, ParamValue> | null = null;
     try {
       // resolve references in the input
-      result = await execute(resolvedInput, previousState.input);
+      const nodeOptions = { prevInput: previousState.input, globals: runOptions?.globals || {} };
+      result = await execute(resolvedInput, nodeOptions);
       runOptions?.onNodeRunComplete?.(nodeId, result);
       logger.debug({ nodeId, result }, "Node run complete");
       // track the run
@@ -147,7 +148,7 @@ export async function runFlow(flow: Flow, nodeId: string, input: Record<string,
     if (runOptions?.mode === "dataflow") return;
 
     // define the execution order
-    const executionOrder = await getTargets(resolvedInput, previousState.input, result);
+    const executionOrder = await getTargets(resolvedInput, result, { prevInput: previousState.input });
     const executionTargets = targets.filter(
       (edge) => edge.sourceHandle.startsWith(executionPrefix) && edge.source === nodeId
     );

diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/updateNodeInput.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/updateNodeInput.ts
@@ -30,7 +30,7 @@ export async function updateNodeInput(
   // parse the input
   const nodeState = flowState[nodeId];
   const prevInput = nodeState?.input;
-  const parsedInput = await mapInput(input, prevInput);
+  const parsedInput = await mapInput(input, { prevInput, globals: runOptions?.globals });
 
   // compile the new state of the node
   const newInput = { ...applyDefaultInputs(prevInput, nodeDefinition), ...parsedInput };

diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/utils/resolveVariables.test.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/utils/resolveVariables.test.ts
@@ -1,6 +1,6 @@
 import { resolveVariables } from "./resolveVariables";
 
-test.only("resolveVariables - extract variables", async () => {
+test("resolveVariables - extract variables", async () => {
   const resolver = jest.fn(async (variableName) => {
     return `RESOLVED_WITH_${variableName}`;
   });
@@ -32,3 +32,64 @@ test.only("resolveVariables - extract variables", async () => {
     }
   `);
 });
+
+test("resolveVariables - handles special chars", async () => {
+  const resolver = jest.fn(async (variableName) => {
+    return `RESOLVED_WITH_${variableName} 
+    
+    "I'm on a separate line"
+    
+`;
+  });
+
+  expect(
+    await resolveVariables(
+      {
+        "param-1": "not_a_variable",
+        "param-2": "${{ps:ref:variable-1}}",
+        "param-3": "${{ps:ref:variable-2}}",
+        "param-4": "{{}}",
+        "param-5": "${{ps:ref:variable-1}} ${{ps:ref:variable-2}}",
+        "param-6": "{{dont_replace}} ${{ps:ref:variable-2}}",
+        "param-7": "{{faulty}",
+        "param-8": "{{faulty} ${{ps:ref:variable-2}}",
+      },
+      resolver
+    )
+  ).toMatchInlineSnapshot(`
+    {
+      "param-1": "not_a_variable",
+      "param-2": "RESOLVED_WITH_variable-1 
+        
+        "I'm on a separate line"
+        
+    ",
+      "param-3": "RESOLVED_WITH_variable-2 
+        
+        "I'm on a separate line"
+        
+    ",
+      "param-4": "{{}}",
+      "param-5": "RESOLVED_WITH_variable-1 
+        
+        "I'm on a separate line"
+        
+     RESOLVED_WITH_variable-2 
+        
+        "I'm on a separate line"
+        
+    ",
+      "param-6": "{{dont_replace}} RESOLVED_WITH_variable-2 
+        
+        "I'm on a separate line"
+        
+    ",
+      "param-7": "{{faulty}",
+      "param-8": "{{faulty} RESOLVED_WITH_variable-2 
+        
+        "I'm on a separate line"
+        
+    ",
+    }
+  `);
+});
diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/utils/resolveVariables.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/utils/resolveVariables.ts
@@ -1,5 +1,4 @@
 import { ParamValue } from "@pufflig/ps-types";
-import Mustache from "mustache";
 import { delimiterEnd, delimiterStart } from "../constants";
 import { extractVariables } from "./extractVariables";
 
@@ -24,15 +23,18 @@ export const resolveVariables = async (
   vars.forEach((variable) => promises.push(resolver(variable)));
   const resolvedVariables = await Promise.all(promises);
 
-  // map of variables with resolved variables
-  const variablesMap = vars.reduce((acc, variable, index) => {
-    acc[variable] = resolvedVariables[index];
-    return acc;
-  }, {} as Record<string, string>);
+  // replace variables with resolved variables, text might contain special characters
+  const result = vars.reduce((acc, name, index) => {
+    const regexString = "\\" + delimiterStart + name + delimiterEnd;
+    const regex = new RegExp(regexString, "g");
+    const value = resolvedVariables[index];
+    return Object.entries(acc).reduce((acc2, [key, val]) => {
+      return {
+        ...acc2,
+        [key]: typeof val === "string" ? val?.replace(regex, value) : val,
+      };
+    }, {} as Record<string, ParamValue>);
+  }, input);
 
-  // insert resolved variables into input
-  const newInput: Record<string, ParamValue> = JSON.parse(
-    Mustache.render(JSON.stringify(input), variablesMap, {}, [delimiterStart, delimiterEnd])
-  );
-  return newInput;
+  return result;
 };
diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/utils/sanitizeInput.test.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/utils/sanitizeInput.test.ts
@@ -0,0 +1,19 @@
+import { sanitizeInput } from "./sanitizeInput";
+
+test("sanitizeInput - remove variables", async () => {
+  const variables = await sanitizeInput({
+    "param-1": "not_a_variable",
+    "param-2": "${{ps:ref:variable-1}}",
+    "param-3": "${{ps:ref:variable-2}",
+    "param-4": "${{variable-4}}",
+  });
+
+  expect(variables).toMatchInlineSnapshot(`
+    {
+      "param-1": "not_a_variable",
+      "param-2": "",
+      "param-3": "\${{ps:ref:variable-2}",
+      "param-4": "\${{variable-4}}",
+    }
+  `);
+});
diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/utils/sanitizeInput.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/utils/sanitizeInput.ts
@@ -0,0 +1,14 @@
+import { ParamValue } from "@pufflig/ps-types";
+import { resolveVariables } from "./resolveVariables";
+
+/**
+ * Remove all variables from an input object
+ *
+ * @param input
+ * @returns
+ */
+export const sanitizeInput = (input: Record<string, ParamValue>) => {
+  return resolveVariables(input, async () => {
+    return "";
+  });
+};
diff --git a/packages/@pufflig/ps-chains/src/engines/dataflow/utils/utils.ts b/packages/@pufflig/ps-chains/src/engines/dataflow/utils/utils.ts
@@ -21,7 +21,7 @@ export function paramToDefaults(params: Param[]) {
  * @returns node state with default values
  */
 export function applyDefaultInputs(nodeStateData: Record<string, ParamValue> | undefined, node: NodeConfig) {
-  const inputs = [...node.inputs, ...node.parameters];
+  const inputs = [...node.inputs];
   const defaults = paramToDefaults(inputs);
   return { ...defaults, ...(nodeStateData || {}) };
 }

diff --git a/packages/@pufflig/ps-chains/src/index.ts b/packages/@pufflig/ps-chains/src/index.ts
@@ -1,3 +1,5 @@
 export { runFlow } from "./engines/dataflow/runFlow";
 export { updateNodeInput } from "./engines/dataflow/updateNodeInput";
 export * from "./types";
+export { resolveVariables } from "./engines/dataflow/utils/resolveVariables";
+export { sanitizeInput } from "./engines/dataflow/utils/sanitizeInput";
diff --git a/packages/@pufflig/ps-chains/src/mocks/chains.ts b/packages/@pufflig/ps-chains/src/mocks/chains.ts
@@ -18,9 +18,6 @@ export const singleNodeFlow: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },
@@ -42,9 +39,6 @@ export const configOnlyFlow: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },
@@ -74,16 +68,10 @@ export const simpleFlow: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
       n2: {
         id: "n2",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },
@@ -108,9 +96,6 @@ export const simpleLoop: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },
@@ -164,23 +149,14 @@ export const mappedExample: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
       n2: {
         id: "n2",
         type: "multi_input",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
       n3: {
         id: "n3",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },
@@ -444,16 +420,10 @@ export const simpleFlowWithVars: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
       n2: {
         id: "n2",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },
@@ -492,23 +462,14 @@ export const simpleFlowWithExec: Flow = {
       n1: {
         id: "n1",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
       n2: {
         id: "n2",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
       n3: {
         id: "n3",
         type: "simple_node",
-        editor: {
-          position: { x: 0, y: 0 },
-        },
       },
     },
   },