Merge pull request #246 from Informatiqal/241-loops

closes #241

package.json

@@ -59,25 +59,25 @@
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-typescript": "^11.1.6",
-    "@types/node": "20.14.2",
-    "@types/uuid": "^9.0.8",
+    "@types/node": "20.14.10",
+    "@types/uuid": "^10.0.0",
     "dotenv": "16.4.5",
     "esm": "^3.2.25",
-    "nyc": "15.1.0",
+    "nyc": "17.0.0",
     "rollup-plugin-delete": "2.0.0",
     "ts-node": "10.9.2",
     "tslib": "2.6.3",
-    "typedoc": "0.25.13",
+    "typedoc": "0.26.4",
     "typescript": "^5.0.4",
     "vitest": "^1.5.1"
   },
   "dependencies": {
-    "@informatiqal/automatiqal-schema": "^0.9.7",
+    "@informatiqal/automatiqal-schema": "^0.10.2",
     "ajv": "^8.16.0",
     "ajv-errors": "^3.0.0",
     "events": "^3.3.0",
-    "qlik-repo-api": "^0.12.3",
+    "qlik-repo-api": "^0.13.1",
     "qlik-saas-api": "^0.18.3",
-    "rollup": "^4.18.0"
+    "rollup": "^4.18.1"
   }
 }

src/RunBook/RunBook.interfaces.ts

@@ -255,6 +255,11 @@ export type WinOperations =
 
 export type TAddRemoveSet = "add" | "remove" | "set";
 
+export type ILoop =
+  | string
+  | number
+  | { [k: string]: string | number | boolean };
+
 export interface ITask {
   name: string;
   description?: string;
@@ -273,6 +278,7 @@ export interface ITask {
     tagOperation?: TAddRemoveSet;
     customPropertyOperation?: TAddRemoveSet;
     unmaskSecrets?: boolean;
+    loopParallel?: boolean;
   };
   location?: string;
   details?: TaskDetails;
@@ -282,6 +288,7 @@ export interface ITask {
     ignore?: boolean;
     tasks?: ITask[];
   };
+  loop?: ILoop[];
 }
 
 export type TraceLevels = "error" | "debug";

src/RunBook/Runner.ts

@@ -1,6 +1,6 @@
 import { QlikRepoApi } from "qlik-repo-api";
 import { randomUUID } from "crypto";
-import { IRunBook, ITask } from "./RunBook.interfaces";
+import { ILoop, IRunBook, ITask } from "./RunBook.interfaces";
 import { Task } from "./Task";
 import { Debugger } from "../util/Debugger";
 import { EventsBus } from "../util/EventBus";
@@ -78,13 +78,12 @@ export class Runner {
       a[0] = `${a[0]}s`;
     }
 
-    const taskOperationMeta = this.operations.ops.filter(task.operation)
+    const taskOperationMeta = this.operations.ops.filter(task.operation);
 
-    const data = 
-    await this.instance[`${a[0]}`].getFilter({
+    const data = await this.instance[`${a[0]}`].getFilter({
       filter: task.filter,
     });
-    
+
     // taskOperationMeta.realOperation ?await this.instance
 
     // this.debug.print(task.name, data.length);
@@ -93,6 +92,8 @@ export class Runner {
 
   private async taskProcessing(t: ITask) {
     if (!t.operation) throw new CustomError(1012, t.name, { arg1: t.name });
+    if (!t.loop) t.loop = [];
+
     try {
       // TODO: data should have type!
       // get the data for the object on which the operation will be performed
@@ -121,42 +122,55 @@ export class Runner {
         totalSeconds: -1,
       };
 
-      const regex1 = new RegExp(/(?<=\$\${)(.*?)(?=})/gm);
       // check for inline variables inside the task details
+      const regex1 = new RegExp(/(?<=\$\${)(.*?)(?=})/gm);
       if (t.details && regex1.test(JSON.stringify(t.details))) {
         t.details = this.replaceInlineVariables(t.details, t.name);
       }
 
       t = this.replaceSpecialVariables(t);
 
-      const task = new Task(t, this.instance, data);
+      const result: ITaskResult = {
+        task: this.maskSensitiveData(t),
+        timings: {} as ITaskTimings,
+        data: {} as any,
+        status: "Completed",
+      };
+
       timings.start = new Date().toISOString();
 
-      await task.process().then((taskResult) => {
-        timings.end = new Date().toISOString();
-        timings.totalSeconds =
-          (new Date(timings.end).getTime() -
-            new Date(timings.start).getTime()) /
-          1000;
-
-        const result: ITaskResult = {
-          task: this.maskSensitiveData(t),
-          timings: timings,
-          // by default all sensitive data will be automatically masked
-          // unless the task options specifically disables this behavior
-          // aka: options.unmaskSecrets == true
-          data:
-            t.options && t.options?.unmaskSecrets == true
-              ? taskResult
-              : this.maskSensitiveDataDetails(taskResult, t.operation),
-          status: "Completed",
-        };
-        this.emitter.emit("task:result", result);
-        this.emitter.emit("runbook:log", `${task.task.name} complete`);
-        this.taskResults.push(result);
-        this.emitter.emit("runbook:result", this.taskResults);
-        return result;
-      });
+      let taskResults: IRunBookResult[] = [];
+
+      // loop through all values and execute the task again
+      if (t.options?.loopParallel == true) {
+        taskResults = await Promise.all(
+          t.loop.map((loopValue, i) => {
+            return this.runTaskLoop(t, i, data, loopValue);
+          })
+        ).then((r) => r.flat());
+      } else {
+        let taskResultPostLoop: IRunBookResult[][] = [];
+
+        for (let i = 0; i < t.loop.length; i++) {
+          const loopedData = await this.runTaskLoop(t, i, data, t.loop[i]);
+          taskResultPostLoop.push(loopedData);
+        }
+
+        taskResults = taskResultPostLoop.flat();
+      }
+
+      timings.end = new Date().toISOString();
+      timings.totalSeconds =
+        (new Date(timings.end).getTime() - new Date(timings.start).getTime()) /
+        1000;
+
+      result.timings = timings;
+      result.data = taskResults.flat();
+
+      this.emitter.emit("task:result", result);
+      this.emitter.emit("runbook:log", `${t.name} complete`);
+      this.taskResults.push(result);
+      this.emitter.emit("runbook:result", this.taskResults);
     } catch (e) {
       this.taskResults.push({
         task: this.maskSensitiveData(t),
@@ -187,6 +201,80 @@ export class Runner {
     }
   }
 
+  private async runTaskLoop(t: ITask, i: number, data: any, loopValue: ILoop) {
+    const taskWithReplacedLoopVariables = this.replaceLoopVariablesInTask(
+      t,
+      loopValue,
+      i
+    );
+    const task = new Task(taskWithReplacedLoopVariables, this.instance, data);
+
+    const taskResult = await task.process();
+
+    // by default all sensitive data will be automatically masked
+    // unless the task options specifically disables this behavior
+    // aka: options.unmaskSecrets == true
+    const dataToReturn =
+      t.options && t.options?.unmaskSecrets == true
+        ? taskResult
+        : this.maskSensitiveDataDetails(taskResult, t.operation);
+
+    return dataToReturn;
+  }
+
+  private replaceLoopVariablesInTask(
+    task: ITask,
+    loopValue: ILoop,
+    index: number
+  ) {
+    // get all instances of something between {{ and }}
+    const regex = new RegExp(/(?<={{)(.*?)(?=}})/gm);
+    // make the task string
+    const taskStringTemplate = JSON.stringify(task);
+    // find all instances of loop variables into the stringified task
+    const loopVariables = [...taskStringTemplate.matchAll(regex)];
+
+    // if loop is defined but not loop variables are found - throw an error
+    if (loopVariables.length == 0)
+      throw new CustomError(1027, "RunBook", {
+        arg1: task.name,
+      });
+
+    let taskString = taskStringTemplate;
+
+    // loop through all loop variables instances
+    loopVariables.map((v) => {
+      const value = v[0].trim();
+      // if index then replace the content with the current loop index
+      if (value == "index")
+        taskString = taskString.replaceAll(`{{${v[0]}}}`, index as any);
+
+      // if item then replace the content with the current loop value
+      if (value == "item")
+        taskString = taskString.replaceAll(`{{${v[0]}}}`, loopValue as any);
+
+      // if starts with item. then the loop values are in object format
+      if (value.startsWith("item.")) {
+        // get the property name
+        const b = value.replace("item.", "");
+
+        // if the current loop value dont have the provided property - throw an error
+        if (!loopValue.hasOwnProperty(b))
+          throw new CustomError(1028, "RunBook", {
+            arg1: task.name,
+            arg2: b,
+          });
+
+        // replace all instances with the loop value property
+        taskString = taskString.replaceAll(`{{${v[0]}}}`, loopValue[b]);
+      }
+    });
+
+    const newTask = JSON.parse(taskString) as ITask;
+
+    return newTask;
+  }
+
   // if at least one of the details prop values
   // is an inline variable $${xxx} then
   // replace its value with the id(s) from

src/index.ts

@@ -7,7 +7,7 @@ import {
 import Ajv, { ValidateFunction } from "ajv";
 import ajvErrors from "ajv-errors";
 
-import { IRunBookResult, ITaskResult, Runner } from "./RunBook/Runner";
+import { ITaskResult, Runner } from "./RunBook/Runner";
 import { IRunBook, ITask } from "./RunBook/RunBook.interfaces";
 import { CustomError } from "./util/CustomError";
 import { EventsBus } from "./util/EventBus";

src/util/CustomError.ts

@@ -44,6 +44,8 @@ export class CustomError extends Error {
     1024: `Source for inline task contains more than one ID. Task %{arg1}. Inline variable %{arg2}`,
     1025: `Task names should not contain # symbol. Please rename the following task(s): %{arg1}`,
     1026: `Missing/incorrect property when extracting inline variable value from task "%{arg1}" - %{arg2}`,
+    1027: `Loop property is present in task "%{arg1}" but no loop variables are used. Either use the variables or remove the loop.`,
+    1028: `Property "%{arg2}" to not exists in loop values. In task "%{arg1}"`,
   };
 
   constructor(code: number, taskName: string, params?: IErrorParams) {

tsconfig.json

@@ -1,7 +1,7 @@
 {
     "compilerOptions": {
         "outDir": "./dist",
-        "target": "ES6",
+        "target": "ES2021",
         "declaration": true,
         "module": "ES6",
         "noImplicitAny": false,